from pyspark import SparkContext, SparkConf, SQLContext
from pyspark.sql import functions as F
import pandas as pd
from pyspark.sql.types import IntegerType
import numpy as np

sc = SparkContext("local", "bike")
sqlContext = SQLContext(sc)
df = sqlContext.read.format('com.databricks.spark.csv').options(header='true', inferschema='true').load('201904-citibike-tripdata.csv')
df5 = sqlContext.read.format('com.databricks.spark.csv').options(header='true', inferschema='true').load('201905-citibike-tripdata.csv')
df6 = sqlContext.read.format('com.databricks.spark.csv').options(header='true', inferschema='true').load('201906-citibike-tripdata.csv')
df7 = sqlContext.read.format('com.databricks.spark.csv').options(header='true', inferschema='true').load('201907-citibike-tripdata.csv')
df=df.union(df5)
df=df.union(df6)
df=df.union(df7)

df = df.filter(df['start station id']==519)

hour_window = F.window(F.col("starttime"), "1 hour", "1 hour").start.alias("starttime")
hour_num = df.groupBy(hour_window).count()
print(hour_num.count())
hour_num.show()

2563
+-------------------+-----+
|          starttime|count|
+-------------------+-----+
|2019-04-02 23:00:00|    5|
|2019-04-13 19:00:00|    8|
|2019-04-14 12:00:00|   16|
|2019-04-22 22:00:00|   11|
|2019-05-02 04:00:00|    1|
|2019-05-31 10:00:00|   25|
|2019-06-08 01:00:00|    1|
|2019-06-17 20:00:00|   12|
|2019-07-03 17:00:00|   93|
|2019-04-19 16:00:00|   38|
|2019-05-02 21:00:00|   10|
|2019-05-04 04:00:00|    1|
|2019-05-31 22:00:00|    5|
|2019-05-31 23:00:00|    8|
|2019-06-16 07:00:00|    1|
|2019-06-22 19:00:00|    4|
|2019-06-24 18:00:00|  105|
|2019-07-15 08:00:00|   92|
|2019-07-18 10:00:00|    7|
|2019-04-03 04:00:00|    1|
+-------------------+-----+
only showing top 20 rows

df_weather = sqlContext.read.format('com.databricks.spark.csv').options(header='true', inferschema='true').load('processed_weather.csv')
print(df_weather.count())
df_weather.show()

2924
+-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+
|          starttime|                SPD|                GUS|CLR|SCT|BKN|OVC|OBS|POB|               VSB|               TEMP|               DEWP|                 SLP|                 ALT|                 STP|               PCP01|
+-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+
|2019-04-01 00:00:00| 0.4111950573077732| 3.1687885314639845|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.202718242869458|-2.4059857964589435|-0.00771000909058...|-0.03138250547365...|-0.02748200443494122|-0.17811912685987313|
|2019-04-01 01:00:00| 2.8743640151934144| 3.3341854908713513|  0|  1|  0|  0|  0|  0|0.4638614904960641| -2.367325382057515|-2.5515963785584366| 0.10097350748358193| 0.07409157039460014| 0.06639344062031857|-0.17811912685987313|
|2019-04-01 02:00:00| 1.9506756559862992|  2.672597653241884|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.531932521245572|-2.6244016696081833|  0.1630783740973855| 0.12682860832873663|  0.1289770706571525|-0.17811912685987313|
|2019-04-01 03:00:00| 1.9506756559862992|  2.672597653241884|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.6142360908396003|-2.6244016696081833|  0.2562356740180996| 0.23230268419699088|  0.2384984232216252|-0.17811912685987313|
|2019-04-01 04:00:00| 1.6427795362505941|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296|  0.4270240572060726|  0.3905137979993629| 0.39495749831371896|-0.17811912685987313|
|2019-04-01 05:00:00| 1.3348834165148888| 3.0033915720566178|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.696539660433629|-2.6972069606579296|  0.5822862237405904|  0.5487249118017536|  0.5514165734058126|-0.17811912685987313|
|2019-04-01 06:00:00| 1.3348834165148888| 3.6649794096860853|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.7788432300276575| -2.770012251707676|  0.7220221736216527|  0.7069360256041444|  0.7078756484979064|-0.17811912685987313|
|2019-04-01 07:00:00| 1.9506756559862992| 3.3341854908713513|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.7788432300276575| -2.770012251707676|  0.9238629901165186|  0.9178841773406529|  0.9112724461176211|-0.17811912685987313|
|2019-04-01 08:00:00| 1.3348834165148888| 2.5072006938345175|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296|  1.0014940733837774|  0.9706212152747894|   0.973856076154455|-0.17811912685987313|
|2019-04-01 09:00:00| 1.3348834165148888|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.4496289516515435|-2.6972069606579296|  1.1257038066114022|  1.0760952911430437|  1.0833774287189277|-0.17811912685987313|
|2019-04-01 10:00:00| 1.0269872967791835| 3.6649794096860853|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.2850218124634862|-2.6972069606579296|  1.2499135398390093|  1.2343064049454158|  1.2398365038110215|-0.17811912685987313|
|2019-04-01 11:00:00| 1.3348834165148888| 3.6649794096860853|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.202718242869458|-2.6972069606579296|  1.3120184064528129|   1.287043442879552|  1.2867742263386424|-0.17811912685987313|
|2019-04-01 12:00:00| 2.2585717757220043| 3.1687885314639845|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.038111103681401|-2.8428175427574227|  1.3585970564131788|    1.33978048081367|  1.3337119488662812|-0.17811912685987313|
|2019-04-01 13:00:00|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.8735039644933438| -2.915622833807169|  1.3585970564131788|    1.33978048081367|  1.3337119488662812|-0.17811912685987313|
|2019-04-01 14:00:00|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-2.9884281248569153|  1.3741232730666164|    1.33978048081367|  1.3337119488662812|-0.17811912685987313|
|2019-04-01 15:00:00| 1.9506756559862992| 3.0033915720566178|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-3.1340387069564084|  1.3896494897200717|  1.3925175187478065|  1.3962955789031153|-0.17811912685987313|
|2019-04-01 16:00:00| 1.9506756559862992|  2.672597653241884|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-3.2068439980061547|  1.4362281396804375|  1.4452545566819428|  1.4432333014307361|-0.17811912685987313|
|2019-04-01 17:00:00| 2.2585717757220043| 3.1687885314639845|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-3.2796492890559015|  1.5914903062149552|  1.5507286325501972|  1.5527546539952088|-0.17811912685987313|
|2019-04-01 18:00:00| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.8735039644933438|-3.2068439980061547|  1.7157000394425623|   1.708939746352569|  1.7092137290873026|-0.17811912685987313|
|2019-04-01 19:00:00| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.9558075340873724|-3.2796492890559015|  1.8399097726701694|  1.8144138222208235|  1.8187350816517576|-0.17811912685987313|
+-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+
only showing top 20 rows

df_join = df_weather.join(hour_num, ['starttime'], "left")
df_join = df_join.withColumn("count", df_join["count"].cast(IntegerType()))
df_new = df_join.fillna(0, subset='count')
df_final=df_new.orderBy('starttime')
print(df_final.count())
df_final.show()

2924
+-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+
|          starttime|                SPD|                GUS|CLR|SCT|BKN|OVC|OBS|POB|               VSB|               TEMP|               DEWP|                 SLP|                 ALT|                 STP|               PCP01|count|
+-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+
|2019-04-01 00:00:00| 0.4111950573077732| 3.1687885314639845|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.202718242869458|-2.4059857964589435|-0.00771000909058...|-0.03138250547365...|-0.02748200443494122|-0.17811912685987313|    5|
|2019-04-01 01:00:00| 2.8743640151934144| 3.3341854908713513|  0|  1|  0|  0|  0|  0|0.4638614904960641| -2.367325382057515|-2.5515963785584366| 0.10097350748358193| 0.07409157039460014| 0.06639344062031857|-0.17811912685987313|    0|
|2019-04-01 02:00:00| 1.9506756559862992|  2.672597653241884|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.531932521245572|-2.6244016696081833|  0.1630783740973855| 0.12682860832873663|  0.1289770706571525|-0.17811912685987313|    0|
|2019-04-01 03:00:00| 1.9506756559862992|  2.672597653241884|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.6142360908396003|-2.6244016696081833|  0.2562356740180996| 0.23230268419699088|  0.2384984232216252|-0.17811912685987313|    0|
|2019-04-01 04:00:00| 1.6427795362505941|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296|  0.4270240572060726|  0.3905137979993629| 0.39495749831371896|-0.17811912685987313|    0|
|2019-04-01 05:00:00| 1.3348834165148888| 3.0033915720566178|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.696539660433629|-2.6972069606579296|  0.5822862237405904|  0.5487249118017536|  0.5514165734058126|-0.17811912685987313|    3|
|2019-04-01 06:00:00| 1.3348834165148888| 3.6649794096860853|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.7788432300276575| -2.770012251707676|  0.7220221736216527|  0.7069360256041444|  0.7078756484979064|-0.17811912685987313|   16|
|2019-04-01 07:00:00| 1.9506756559862992| 3.3341854908713513|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.7788432300276575| -2.770012251707676|  0.9238629901165186|  0.9178841773406529|  0.9112724461176211|-0.17811912685987313|   39|
|2019-04-01 08:00:00| 1.3348834165148888| 2.5072006938345175|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296|  1.0014940733837774|  0.9706212152747894|   0.973856076154455|-0.17811912685987313|   63|
|2019-04-01 09:00:00| 1.3348834165148888|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.4496289516515435|-2.6972069606579296|  1.1257038066114022|  1.0760952911430437|  1.0833774287189277|-0.17811912685987313|   38|
|2019-04-01 10:00:00| 1.0269872967791835| 3.6649794096860853|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.2850218124634862|-2.6972069606579296|  1.2499135398390093|  1.2343064049454158|  1.2398365038110215|-0.17811912685987313|   18|
|2019-04-01 11:00:00| 1.3348834165148888| 3.6649794096860853|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.202718242869458|-2.6972069606579296|  1.3120184064528129|   1.287043442879552|  1.2867742263386424|-0.17811912685987313|    7|
|2019-04-01 12:00:00| 2.2585717757220043| 3.1687885314639845|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.038111103681401|-2.8428175427574227|  1.3585970564131788|    1.33978048081367|  1.3337119488662812|-0.17811912685987313|   10|
|2019-04-01 13:00:00|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.8735039644933438| -2.915622833807169|  1.3585970564131788|    1.33978048081367|  1.3337119488662812|-0.17811912685987313|   16|
|2019-04-01 14:00:00|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-2.9884281248569153|  1.3741232730666164|    1.33978048081367|  1.3337119488662812|-0.17811912685987313|   17|
|2019-04-01 15:00:00| 1.9506756559862992| 3.0033915720566178|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-3.1340387069564084|  1.3896494897200717|  1.3925175187478065|  1.3962955789031153|-0.17811912685987313|   22|
|2019-04-01 16:00:00| 1.9506756559862992|  2.672597653241884|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-3.2068439980061547|  1.4362281396804375|  1.4452545566819428|  1.4432333014307361|-0.17811912685987313|   32|
|2019-04-01 17:00:00| 2.2585717757220043| 3.1687885314639845|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-3.2796492890559015|  1.5914903062149552|  1.5507286325501972|  1.5527546539952088|-0.17811912685987313|   86|
|2019-04-01 18:00:00| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.8735039644933438|-3.2068439980061547|  1.7157000394425623|   1.708939746352569|  1.7092137290873026|-0.17811912685987313|  133|
|2019-04-01 19:00:00| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.9558075340873724|-3.2796492890559015|  1.8399097726701694|  1.8144138222208235|  1.8187350816517576|-0.17811912685987313|   40|
+-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+
only showing top 20 rows

from pyspark.sql.types import *
from pyspark.sql import *
from pyspark.sql.types import StructType, StructField, LongType, StringType
from pyspark.sql import Row
from pyspark.sql import Column
import pandas as pd
import numpy as np
import datetime

spark=SparkSession \
.builder \
.appName('my_app_name') \
.getOrCreate()

#here we add more features: hour and wheter it is weekday

pd_final = df_final.toPandas()
pd_hour = [x.hour for x in pd_final['starttime']]
pd_final['hour'] = pd_hour

pd_weekday = [x.weekday() for x in pd_final['starttime']]
pd_final['weekday'] = pd_weekday
pd_final['weekday_end'] = ((pd_final['weekday']//5 == 1).astype(int))
pd_final[pd_final['weekday']==6]
pd_final.drop(columns = ['weekday'], inplace=True)

cols = pd_final.columns
column_names = ['starttime', 'SPD', 'GUS', 'CLR', 'SCT', 'BKN', 'OVC', 'OBS', 'POB',
       'VSB', 'TEMP', 'DEWP', 'SLP', 'ALT', 'STP', 'PCP01', 'hour',
       'weekday_end', 'count']
pd_final = pd_final[column_names]
df_spark = spark.createDataFrame(pd_final)
df_spark.show()

+-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+----+-----------+-----+
|          starttime|                SPD|                GUS|CLR|SCT|BKN|OVC|OBS|POB|               VSB|               TEMP|               DEWP|                 SLP|                 ALT|                 STP|               PCP01|hour|weekday_end|count|
+-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+----+-----------+-----+
|2019-04-01 00:00:00| 0.4111950573077732| 3.1687885314639845|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.202718242869458|-2.4059857964589435|-0.00771000909058...|-0.03138250547365...|-0.02748200443494122|-0.17811912685987313|   0|          0|    5|
|2019-04-01 01:00:00| 2.8743640151934144| 3.3341854908713513|  0|  1|  0|  0|  0|  0|0.4638614904960641| -2.367325382057515|-2.5515963785584366| 0.10097350748358193| 0.07409157039460014| 0.06639344062031857|-0.17811912685987313|   1|          0|    0|
|2019-04-01 02:00:00| 1.9506756559862992|  2.672597653241884|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.531932521245572|-2.6244016696081833|  0.1630783740973855| 0.12682860832873663|  0.1289770706571525|-0.17811912685987313|   2|          0|    0|
|2019-04-01 03:00:00| 1.9506756559862992|  2.672597653241884|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.6142360908396003|-2.6244016696081833|  0.2562356740180996| 0.23230268419699088|  0.2384984232216252|-0.17811912685987313|   3|          0|    0|
|2019-04-01 04:00:00| 1.6427795362505941|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296|  0.4270240572060726|  0.3905137979993629| 0.39495749831371896|-0.17811912685987313|   4|          0|    0|
|2019-04-01 05:00:00| 1.3348834165148888| 3.0033915720566178|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.696539660433629|-2.6972069606579296|  0.5822862237405904|  0.5487249118017536|  0.5514165734058126|-0.17811912685987313|   5|          0|    3|
|2019-04-01 06:00:00| 1.3348834165148888| 3.6649794096860853|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.7788432300276575| -2.770012251707676|  0.7220221736216527|  0.7069360256041444|  0.7078756484979064|-0.17811912685987313|   6|          0|   16|
|2019-04-01 07:00:00| 1.9506756559862992| 3.3341854908713513|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.7788432300276575| -2.770012251707676|  0.9238629901165186|  0.9178841773406529|  0.9112724461176211|-0.17811912685987313|   7|          0|   39|
|2019-04-01 08:00:00| 1.3348834165148888| 2.5072006938345175|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296|  1.0014940733837774|  0.9706212152747894|   0.973856076154455|-0.17811912685987313|   8|          0|   63|
|2019-04-01 09:00:00| 1.3348834165148888|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.4496289516515435|-2.6972069606579296|  1.1257038066114022|  1.0760952911430437|  1.0833774287189277|-0.17811912685987313|   9|          0|   38|
|2019-04-01 10:00:00| 1.0269872967791835| 3.6649794096860853|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.2850218124634862|-2.6972069606579296|  1.2499135398390093|  1.2343064049454158|  1.2398365038110215|-0.17811912685987313|  10|          0|   18|
|2019-04-01 11:00:00| 1.3348834165148888| 3.6649794096860853|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.202718242869458|-2.6972069606579296|  1.3120184064528129|   1.287043442879552|  1.2867742263386424|-0.17811912685987313|  11|          0|    7|
|2019-04-01 12:00:00| 2.2585717757220043| 3.1687885314639845|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.038111103681401|-2.8428175427574227|  1.3585970564131788|    1.33978048081367|  1.3337119488662812|-0.17811912685987313|  12|          0|   10|
|2019-04-01 13:00:00|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.8735039644933438| -2.915622833807169|  1.3585970564131788|    1.33978048081367|  1.3337119488662812|-0.17811912685987313|  13|          0|   16|
|2019-04-01 14:00:00|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-2.9884281248569153|  1.3741232730666164|    1.33978048081367|  1.3337119488662812|-0.17811912685987313|  14|          0|   17|
|2019-04-01 15:00:00| 1.9506756559862992| 3.0033915720566178|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-3.1340387069564084|  1.3896494897200717|  1.3925175187478065|  1.3962955789031153|-0.17811912685987313|  15|          0|   22|
|2019-04-01 16:00:00| 1.9506756559862992|  2.672597653241884|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-3.2068439980061547|  1.4362281396804375|  1.4452545566819428|  1.4432333014307361|-0.17811912685987313|  16|          0|   32|
|2019-04-01 17:00:00| 2.2585717757220043| 3.1687885314639845|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-3.2796492890559015|  1.5914903062149552|  1.5507286325501972|  1.5527546539952088|-0.17811912685987313|  17|          0|   86|
|2019-04-01 18:00:00| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.8735039644933438|-3.2068439980061547|  1.7157000394425623|   1.708939746352569|  1.7092137290873026|-0.17811912685987313|  18|          0|  133|
|2019-04-01 19:00:00| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.9558075340873724|-3.2796492890559015|  1.8399097726701694|  1.8144138222208235|  1.8187350816517576|-0.17811912685987313|  19|          0|   40|
+-------------------+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+----+-----------+-----+
only showing top 20 rows

from pyspark.sql import SparkSession
#split the dataset tp three set, first one is training set, second one is testing set, the last one is for realtime renewing model set
list_final=df_final.collect()
train_list=list_final[0:1464]
test_list=list_final[1464:2184]
realtime_list=list_final[2184:2924] 

a=train_list
b=test_list
c=realtime_list

print(len(b))
print(len(c))
print(a[1400:1464])
print(b[0:50])
print(c[0:50])

720
740
[Row(starttime=datetime.datetime(2019, 5, 29, 8, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.7558606301967902, TEMP=-0.5566468509888874, DEWP=0.2150046813319293, SLP=-1.0169140915649526, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=90), Row(starttime=datetime.datetime(2019, 5, 29, 9, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.5566468509888874, DEWP=0.14219939028218284, SLP=-1.0479665248718455, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=65), Row(starttime=datetime.datetime(2019, 5, 29, 10, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.474343281394859, DEWP=0.14219939028218284, SLP=-1.0169140915649526, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=23), Row(starttime=datetime.datetime(2019, 5, 29, 11, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.1451290030187449, DEWP=0.28780997238167577, SLP=-1.0479665248718455, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=22), Row(starttime=datetime.datetime(2019, 5, 29, 12, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.28780997238167577, SLP=-1.0169140915649526, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=10), Row(starttime=datetime.datetime(2019, 5, 29, 13, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.28780997238167577, SLP=-1.0634927415253008, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=15), Row(starttime=datetime.datetime(2019, 5, 29, 14, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.2150046813319293, SLP=-1.2653335580201843, ALT=-1.2970714158927426, STP=-1.294800512680904, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 5, 29, 15, 0), SPD=1.9506756559862992, GUS=2.3418037344271507, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.2150046813319293, SLP=-1.3895432912477914, ALT=-1.402545491760997, STP=-1.4043218652453588, PCP01=-0.17811912685987313, count=19), Row(starttime=datetime.datetime(2019, 5, 29, 16, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.30973614220680196, DEWP=0.06939409923243636, SLP=-1.3895432912477914, ALT=-1.402545491760997, STP=-1.4043218652453588, PCP01=-0.17811912685987313, count=38), Row(starttime=datetime.datetime(2019, 5, 29, 17, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.7887308313515473, TEMP=-0.6389504205829161, DEWP=0.14219939028218284, SLP=-1.1877024747529255, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=6.460378667254811, count=42), Row(starttime=datetime.datetime(2019, 5, 29, 18, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.7212539901769446, DEWP=0.28780997238167577, SLP=-1.0479665248718455, ALT=-1.2970714158927426, STP=-1.294800512680904, PCP01=4.046379469394926, count=10), Row(starttime=datetime.datetime(2019, 5, 29, 19, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-3.1953048715824988, TEMP=-0.7212539901769446, DEWP=0.28780997238167577, SLP=-1.3119122079805325, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=8.874377865114695, count=11), Row(starttime=datetime.datetime(2019, 5, 29, 20, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.569008710658693, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-1.0324403082184077, ALT=-1.2970714158927426, STP=-1.294800512680904, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 5, 29, 21, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.9755827508896444, TEMP=-0.8858611293650016, DEWP=0.14219939028218284, SLP=-1.0324403082184077, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=11), Row(starttime=datetime.datetime(2019, 5, 29, 22, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.569008710658693, TEMP=-0.8035575597709731, DEWP=0.14219939028218284, SLP=-1.0013878749114973, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=7), Row(starttime=datetime.datetime(2019, 5, 29, 23, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.9755827508896444, TEMP=-0.8035575597709731, DEWP=0.14219939028218284, SLP=-1.0479665248718455, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=4), Row(starttime=datetime.datetime(2019, 5, 30, 0, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.8035575597709731, DEWP=0.14219939028218284, SLP=-1.0634927415253008, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 5, 30, 1, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-1.1100713914856666, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 5, 30, 2, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-1.0634927415253008, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 5, 30, 3, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-1.0324403082184077, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 5, 30, 4, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-1.0013878749114973, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 5, 30, 5, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-0.954809224951149, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=7), Row(starttime=datetime.datetime(2019, 5, 30, 6, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.8035575597709731, DEWP=0.2150046813319293, SLP=-0.8616519250304349, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=25), Row(starttime=datetime.datetime(2019, 5, 30, 7, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.7212539901769446, DEWP=0.2150046813319293, SLP=-0.8461257083769795, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=58), Row(starttime=datetime.datetime(2019, 5, 30, 8, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.5566468509888874, DEWP=0.36061526343142225, SLP=-0.7063897584959171, ALT=-0.7169639986173161, STP=-0.7159019348401501, PCP01=-0.17811912685987313, count=82), Row(starttime=datetime.datetime(2019, 5, 30, 9, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.5566468509888874, DEWP=0.36061526343142225, SLP=-0.7219159751493724, ALT=-0.7169639986173161, STP=-0.7159019348401501, PCP01=-0.17811912685987313, count=72), Row(starttime=datetime.datetime(2019, 5, 30, 10, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.1624346704277415, TEMP=-0.30973614220680196, DEWP=0.36061526343142225, SLP=-0.7529684084562653, ALT=-0.7697010365514526, STP=-0.7628396573677888, PCP01=-0.17811912685987313, count=23), Row(starttime=datetime.datetime(2019, 5, 30, 11, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.1624346704277415, TEMP=-0.22743257261277341, DEWP=0.36061526343142225, SLP=-0.7995470584166313, ALT=-0.7697010365514526, STP=-0.7628396573677888, PCP01=-0.17811912685987313, count=18), Row(starttime=datetime.datetime(2019, 5, 30, 12, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.7558606301967902, TEMP=0.01947813616931213, DEWP=0.43342055448116873, SLP=-0.8616519250304349, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=27), Row(starttime=datetime.datetime(2019, 5, 30, 13, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.3492865899658387, TEMP=0.10178170576334064, DEWP=0.43342055448116873, SLP=-0.9392830082976936, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=20), Row(starttime=datetime.datetime(2019, 5, 30, 14, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.7558606301967902, TEMP=-0.22743257261277341, DEWP=0.36061526343142225, SLP=-1.0169140915649526, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=23), Row(starttime=datetime.datetime(2019, 5, 30, 15, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.7558606301967902, TEMP=-0.22743257261277341, DEWP=0.43342055448116873, SLP=-1.1255976081391044, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=21), Row(starttime=datetime.datetime(2019, 5, 30, 16, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.7558606301967902, TEMP=-0.22743257261277341, DEWP=0.43342055448116873, SLP=-1.2187549080598183, ALT=-1.1915973400244697, STP=-1.1852791601164312, PCP01=-0.17811912685987313, count=45), Row(starttime=datetime.datetime(2019, 5, 30, 17, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.8700456393977376, TEMP=-0.39203971180083047, DEWP=0.5062258455309152, SLP=-1.3119122079805325, ALT=-1.3498084538268604, STP=-1.3573841427177378, PCP01=1.0288804720700693, count=56), Row(starttime=datetime.datetime(2019, 5, 30, 18, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=1, POB=0, VSB=-3.3985918916979743, TEMP=-0.39203971180083047, DEWP=0.5062258455309152, SLP=-1.2808597746736219, ALT=-1.3498084538268604, STP=-1.3573841427177378, PCP01=8.270878065649725, count=4), Row(starttime=datetime.datetime(2019, 5, 30, 19, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.7887308313515473, TEMP=-0.474343281394859, DEWP=0.5062258455309152, SLP=-1.3895432912477914, ALT=-1.2970714158927426, STP=-1.294800512680904, PCP01=2.235880071000012, count=12), Row(starttime=datetime.datetime(2019, 5, 30, 20, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.8700456393977376, TEMP=-0.39203971180083047, DEWP=0.5790311365806616, SLP=-1.3895432912477914, ALT=-1.402545491760997, STP=-1.4043218652453588, PCP01=0.4253806726050981, count=3), Row(starttime=datetime.datetime(2019, 5, 30, 21, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.39203971180083047, DEWP=0.5790311365806616, SLP=-1.3584908579408808, ALT=-1.4552825296951146, STP=-1.4512595877729977, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 5, 30, 22, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.30973614220680196, DEWP=0.6518364276304082, SLP=-1.451648157861595, ALT=-1.402545491760997, STP=-1.4043218652453588, PCP01=-0.17811912685987313, count=4), Row(starttime=datetime.datetime(2019, 5, 30, 23, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.7887308313515473, TEMP=-0.30973614220680196, DEWP=0.6518364276304082, SLP=-1.5137530244753985, ALT=-1.4552825296951146, STP=-1.4512595877729977, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 5, 31, 0, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.8700456393977376, TEMP=-0.1451290030187449, DEWP=0.7974470097299011, SLP=-1.5603316744357645, ALT=-1.4552825296951146, STP=-1.4512595877729977, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 5, 31, 1, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-3.1953048715824988, TEMP=-0.1451290030187449, DEWP=0.7246417186801546, SLP=-1.5292792411288538, ALT=-1.7189677193657595, STP=-1.7172400154295462, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 5, 31, 2, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.7887308313515473, TEMP=-0.1451290030187449, DEWP=0.7974470097299011, SLP=-1.5603316744357645, ALT=-1.5080195676292512, STP=-1.5138432178098316, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 5, 31, 3, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.22743257261277341, DEWP=0.7246417186801546, SLP=-1.5292792411288538, ALT=-1.5607566055633688, STP=-1.5607809403374526, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 5, 31, 4, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.30973614220680196, DEWP=0.6518364276304082, SLP=-1.4671743745150503, ALT=-1.4552825296951146, STP=-1.4512595877729977, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 5, 31, 5, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.569008710658693, TEMP=-0.30973614220680196, DEWP=0.6518364276304082, SLP=-1.3274384246339879, ALT=-1.3498084538268604, STP=-1.3573841427177378, PCP01=-0.17811912685987313, count=6), Row(starttime=datetime.datetime(2019, 5, 31, 6, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.22743257261277341, DEWP=0.6518364276304082, SLP=-1.2187549080598183, ALT=-1.2443343779586062, STP=-1.247862790153265, PCP01=-0.17811912685987313, count=26), Row(starttime=datetime.datetime(2019, 5, 31, 7, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.7246417186801546, SLP=-1.1566500414460148, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=52), Row(starttime=datetime.datetime(2019, 5, 31, 8, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.6518364276304082, SLP=-1.0324403082184077, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=90), Row(starttime=datetime.datetime(2019, 5, 31, 9, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.3486924145454262, DEWP=0.5062258455309152, SLP=-1.0013878749114973, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=64), Row(starttime=datetime.datetime(2019, 5, 31, 10, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.5062258455309152, SLP=-0.9237567916442384, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=25), Row(starttime=datetime.datetime(2019, 5, 31, 11, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.36061526343142225, SLP=-0.8771781416838901, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=19), Row(starttime=datetime.datetime(2019, 5, 31, 12, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.36061526343142225, SLP=-0.8771781416838901, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 5, 31, 13, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.8425138321095973, DEWP=0.2150046813319293, SLP=-0.9392830082976936, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=25), Row(starttime=datetime.datetime(2019, 5, 31, 14, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.2150046813319293, SLP=-1.0169140915649526, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=26), Row(starttime=datetime.datetime(2019, 5, 31, 15, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.2150046813319293, SLP=-1.0945451748322113, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=24), Row(starttime=datetime.datetime(2019, 5, 31, 16, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.14219939028218284, SLP=-1.1100713914856666, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=66), Row(starttime=datetime.datetime(2019, 5, 31, 17, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.2150046813319293, SLP=-1.1100713914856666, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=124), Row(starttime=datetime.datetime(2019, 5, 31, 18, 0), SPD=1.6427795362505941, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.2150046813319293, SLP=-1.0790189581787561, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=100), Row(starttime=datetime.datetime(2019, 5, 31, 19, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.14219939028218284, SLP=-1.0013878749114973, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=40), Row(starttime=datetime.datetime(2019, 5, 31, 20, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5956031233275118, DEWP=0.2150046813319293, SLP=-0.9237567916442384, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=11), Row(starttime=datetime.datetime(2019, 5, 31, 21, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.2150046813319293, SLP=-0.8305994917235242, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=10), Row(starttime=datetime.datetime(2019, 5, 31, 22, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.2150046813319293, SLP=-0.8461257083769795, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 5, 31, 23, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.4309959841394547, DEWP=0.2150046813319293, SLP=-0.8461257083769795, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=8)]
[Row(starttime=datetime.datetime(2019, 6, 1, 0, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.3486924145454262, DEWP=0.28780997238167577, SLP=-0.7995470584166313, ALT=-0.7697010365514526, STP=-0.7628396573677888, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 6, 1, 1, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.2663888449513977, DEWP=0.28780997238167577, SLP=-0.8305994917235242, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=8), Row(starttime=datetime.datetime(2019, 6, 1, 2, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.36061526343142225, SLP=-0.9392830082976936, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=2), Row(starttime=datetime.datetime(2019, 6, 1, 3, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.36061526343142225, SLP=-0.8771781416838901, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 6, 1, 4, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.22743257261277341, DEWP=0.43342055448116873, SLP=-0.7374421918028101, ALT=-0.7169639986173161, STP=-0.7159019348401501, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 6, 1, 5, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.22743257261277341, DEWP=0.28780997238167577, SLP=-0.6287586752286582, ALT=-0.6114899227490619, STP=-0.6063805822756951, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 6, 1, 6, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.22743257261277341, DEWP=0.28780997238167577, SLP=-0.5511275919613994, ALT=-0.5587528848149441, STP=-0.5594428597480563, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 6, 1, 7, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.06939409923243636, SLP=-0.5821800252682924, ALT=-0.6114899227490619, STP=-0.6063805822756951, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 6, 1, 8, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.06939409923243636, SLP=-0.4890227253475959, ALT=-0.5060158468808076, STP=-0.5125051372204353, PCP01=-0.17811912685987313, count=10), Row(starttime=datetime.datetime(2019, 6, 1, 9, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.2663888449513977, DEWP=-0.003411191817310116, SLP=-0.5200751586544888, ALT=-0.5587528848149441, STP=-0.5594428597480563, PCP01=-0.17811912685987313, count=11), Row(starttime=datetime.datetime(2019, 6, 1, 10, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.06939409923243636, SLP=-0.5356013753079442, ALT=-0.5587528848149441, STP=-0.5594428597480563, PCP01=-0.17811912685987313, count=13), Row(starttime=datetime.datetime(2019, 6, 1, 11, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.6779066929215403, DEWP=0.14219939028218284, SLP=-0.5511275919613994, ALT=-0.5587528848149441, STP=-0.5594428597480563, PCP01=-0.17811912685987313, count=20), Row(starttime=datetime.datetime(2019, 6, 1, 12, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.14219939028218284, SLP=-0.6753373251890066, ALT=-0.6642269606831984, STP=-0.668964212312529, PCP01=-0.17811912685987313, count=22), Row(starttime=datetime.datetime(2019, 6, 1, 13, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=-0.003411191817310116, SLP=-0.8150732750700689, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=20), Row(starttime=datetime.datetime(2019, 6, 1, 14, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.14219939028218284, SLP=-0.8771781416838901, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=28), Row(starttime=datetime.datetime(2019, 6, 1, 15, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.8425138321095973, DEWP=0.2150046813319293, SLP=-0.954809224951149, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=19), Row(starttime=datetime.datetime(2019, 6, 1, 16, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.14219939028218284, SLP=-0.8927043583373278, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=9), Row(starttime=datetime.datetime(2019, 6, 1, 17, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.4309959841394547, DEWP=0.2150046813319293, SLP=-0.9392830082976936, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=21), Row(starttime=datetime.datetime(2019, 6, 1, 18, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.2663888449513977, DEWP=0.43342055448116873, SLP=-0.954809224951149, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 6, 1, 19, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.2663888449513977, DEWP=0.36061526343142225, SLP=-0.9082305749907831, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=22), Row(starttime=datetime.datetime(2019, 6, 1, 20, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.0979448542882077, TEMP=0.10178170576334064, DEWP=0.43342055448116873, SLP=-0.8771781416838901, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=10), Row(starttime=datetime.datetime(2019, 6, 1, 21, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.3492865899658387, TEMP=0.01947813616931213, DEWP=0.5062258455309152, SLP=-0.8305994917235242, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=8), Row(starttime=datetime.datetime(2019, 6, 1, 22, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.5062258455309152, SLP=-0.8616519250304349, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=8), Row(starttime=datetime.datetime(2019, 6, 1, 23, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.5062258455309152, SLP=-0.8616519250304349, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=4), Row(starttime=datetime.datetime(2019, 6, 2, 0, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.18408527535736915, DEWP=0.5062258455309152, SLP=-0.9703354416045866, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=4), Row(starttime=datetime.datetime(2019, 6, 2, 1, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-0.3492865899658387, TEMP=0.01947813616931213, DEWP=0.5790311365806616, SLP=-1.0945451748322113, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 6, 2, 2, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.1624346704277415, TEMP=0.10178170576334064, DEWP=0.6518364276304082, SLP=-1.1877024747529255, ALT=-1.1915973400244697, STP=-1.1852791601164312, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 6, 2, 3, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.569008710658693, TEMP=-0.06282543342471639, DEWP=0.6518364276304082, SLP=-1.2653335580201843, ALT=-1.2443343779586062, STP=-1.247862790153265, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 6, 2, 4, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.1624346704277415, TEMP=-0.06282543342471639, DEWP=0.6518364276304082, SLP=-1.2653335580201843, ALT=-1.2970714158927426, STP=-1.294800512680904, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 6, 2, 5, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.382156791120596, TEMP=-0.1451290030187449, DEWP=0.6518364276304082, SLP=-1.1877024747529255, ALT=-1.1915973400244697, STP=-1.1852791601164312, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 6, 2, 6, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.5854438112360714, TEMP=-0.22743257261277341, DEWP=0.6518364276304082, SLP=-1.1721762580994701, ALT=-1.1915973400244697, STP=-1.1852791601164312, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 6, 2, 7, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.9755827508896444, TEMP=0.01947813616931213, DEWP=0.6518364276304082, SLP=-1.1877024747529255, ALT=-1.1915973400244697, STP=-1.1852791601164312, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 6, 2, 8, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-1.9755827508896444, TEMP=0.18408527535736915, DEWP=0.6518364276304082, SLP=-1.1255976081391044, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 6, 2, 9, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.0979448542882077, TEMP=0.4309959841394547, DEWP=0.43342055448116873, SLP=-1.0945451748322113, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=9), Row(starttime=datetime.datetime(2019, 6, 2, 10, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5956031233275118, DEWP=0.36061526343142225, SLP=-1.1100713914856666, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=8), Row(starttime=datetime.datetime(2019, 6, 2, 11, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.8425138321095973, DEWP=0.28780997238167577, SLP=-1.1411238247925597, ALT=-1.1388603020903518, STP=-1.1383414375888101, PCP01=-0.17811912685987313, count=12), Row(starttime=datetime.datetime(2019, 6, 2, 12, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.5062258455309152, SLP=-1.2653335580201843, ALT=-1.2443343779586062, STP=-1.247862790153265, PCP01=-0.17811912685987313, count=13), Row(starttime=datetime.datetime(2019, 6, 2, 13, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=0.2150046813319293, SLP=-1.3895432912477914, ALT=-1.402545491760997, STP=-1.4043218652453588, PCP01=-0.17811912685987313, count=28), Row(starttime=datetime.datetime(2019, 6, 2, 14, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.14219939028218284, SLP=-1.5292792411288538, ALT=-1.5080195676292512, STP=-1.5138432178098316, PCP01=-0.17811912685987313, count=24), Row(starttime=datetime.datetime(2019, 6, 2, 15, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=0.36061526343142225, SLP=-1.5913841077426574, ALT=-1.6134936434975053, STP=-1.6077186628650912, PCP01=-0.17811912685987313, count=20), Row(starttime=datetime.datetime(2019, 6, 2, 16, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=0.36061526343142225, SLP=-1.622436541049568, ALT=-1.6134936434975053, STP=-1.6077186628650912, PCP01=-0.17811912685987313, count=16), Row(starttime=datetime.datetime(2019, 6, 2, 17, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.8700456393977376, TEMP=0.6779066929215403, DEWP=0.5790311365806616, SLP=-1.6534889743564785, ALT=-1.5607566055633688, STP=-1.5607809403374526, PCP01=-0.17811912685987313, count=16), Row(starttime=datetime.datetime(2019, 6, 2, 18, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.0979448542882077, TEMP=0.3486924145454262, DEWP=0.8702523007796475, SLP=-1.6069103243961127, ALT=-1.6134936434975053, STP=-1.6077186628650912, PCP01=-0.17811912685987313, count=18), Row(starttime=datetime.datetime(2019, 6, 2, 19, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.5790311365806616, SLP=-1.5758578910892198, ALT=-1.5607566055633688, STP=-1.5607809403374526, PCP01=-0.17811912685987313, count=4), Row(starttime=datetime.datetime(2019, 6, 2, 20, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.5790311365806616, SLP=-1.4827005911685056, ALT=-1.5080195676292512, STP=-1.5138432178098316, PCP01=-0.17811912685987313, count=12), Row(starttime=datetime.datetime(2019, 6, 2, 21, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-2.8700456393977376, TEMP=0.3486924145454262, DEWP=0.7246417186801546, SLP=-1.0945451748322113, ALT=-1.4552825296951146, STP=-1.4512595877729977, PCP01=-0.17811912685987313, count=8), Row(starttime=datetime.datetime(2019, 6, 2, 22, 0), SPD=0.7190911770434785, GUS=3.1687885314639845, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=-3.4799066997441646, TEMP=-0.22743257261277341, DEWP=0.6518364276304082, SLP=-1.0790189581787561, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=13.702376260834466, count=3), Row(starttime=datetime.datetime(2019, 6, 2, 23, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.39203971180083047, DEWP=0.2150046813319293, SLP=-1.0013878749114973, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 6, 3, 0, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.39203971180083047, DEWP=-0.14902177391680307, SLP=-0.985861658258042, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=2), Row(starttime=datetime.datetime(2019, 6, 3, 1, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.39203971180083047, DEWP=-0.294632356016296, SLP=-1.0013878749114973, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=2)]
[Row(starttime=datetime.datetime(2019, 7, 1, 0, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.18408527535736915, DEWP=0.06939409923243636, SLP=-0.411391642080337, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=2), Row(starttime=datetime.datetime(2019, 7, 1, 1, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.14219939028218284, SLP=-0.3492867754665158, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 7, 1, 2, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.14219939028218284, SLP=-0.2871819088527123, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 7, 1, 3, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.06939409923243636, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 7, 1, 4, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.06939409923243636, SLP=-0.24060325889236406, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 7, 1, 5, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.1451290030187449, DEWP=0.14219939028218284, SLP=-0.20955082558545346, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=8), Row(starttime=datetime.datetime(2019, 7, 1, 6, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.2150046813319293, SLP=-0.19402460893199816, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=16), Row(starttime=datetime.datetime(2019, 7, 1, 7, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.2150046813319293, SLP=-0.14744595897164992, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=52), Row(starttime=datetime.datetime(2019, 7, 1, 8, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.3486924145454262, DEWP=0.36061526343142225, SLP=-0.13191974231819462, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=69), Row(starttime=datetime.datetime(2019, 7, 1, 9, 0), SPD=1.9506756559862992, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.28780997238167577, SLP=-0.08534109235784636, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=54), Row(starttime=datetime.datetime(2019, 7, 1, 10, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.6779066929215403, DEWP=0.36061526343142225, SLP=-0.10086730901130166, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=21), Row(starttime=datetime.datetime(2019, 7, 1, 11, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.16297217562510521, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=15), Row(starttime=datetime.datetime(2019, 7, 1, 12, 0), SPD=1.3348834165148888, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.089424540891683, DEWP=0.2150046813319293, SLP=-0.1784983922785605, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=15), Row(starttime=datetime.datetime(2019, 7, 1, 13, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.2150046813319293, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 7, 1, 14, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.2150046813319293, SLP=-0.3337605588130782, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=12), Row(starttime=datetime.datetime(2019, 7, 1, 15, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.14219939028218284, SLP=-0.3182343421596229, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=27), Row(starttime=datetime.datetime(2019, 7, 1, 16, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.06939409923243636, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=47), Row(starttime=datetime.datetime(2019, 7, 1, 17, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.06939409923243636, SLP=-0.44244407538722996, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=165), Row(starttime=datetime.datetime(2019, 7, 1, 18, 0), SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.47349650869414056, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=128), Row(starttime=datetime.datetime(2019, 7, 1, 19, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.06939409923243636, SLP=-0.4579702920406853, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=36), Row(starttime=datetime.datetime(2019, 7, 1, 20, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.0979448542882077, TEMP=1.089424540891683, DEWP=0.2150046813319293, SLP=-0.4579702920406853, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=18), Row(starttime=datetime.datetime(2019, 7, 1, 21, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.089424540891683, DEWP=0.14219939028218284, SLP=-0.3803392087734264, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=7), Row(starttime=datetime.datetime(2019, 7, 1, 22, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=0.14219939028218284, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=11), Row(starttime=datetime.datetime(2019, 7, 1, 23, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.36061526343142225, SLP=-0.3803392087734264, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=7), Row(starttime=datetime.datetime(2019, 7, 2, 0, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5956031233275118, DEWP=0.36061526343142225, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=2), Row(starttime=datetime.datetime(2019, 7, 2, 1, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.36061526343142225, SLP=-0.411391642080337, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=3), Row(starttime=datetime.datetime(2019, 7, 2, 2, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.43342055448116873, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 7, 2, 3, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.43342055448116873, SLP=-0.4269178587337747, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=2), Row(starttime=datetime.datetime(2019, 7, 2, 4, 0), SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.4309959841394547, DEWP=0.43342055448116873, SLP=-0.47349650869414056, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=0), Row(starttime=datetime.datetime(2019, 7, 2, 5, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.4309959841394547, DEWP=0.36061526343142225, SLP=-0.5511275919613994, ALT=-0.5587528848149441, STP=-0.5594428597480563, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 7, 2, 6, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.4309959841394547, DEWP=0.43342055448116873, SLP=-0.5045489420010335, ALT=-0.5060158468808076, STP=-0.5125051372204353, PCP01=-0.17811912685987313, count=23), Row(starttime=datetime.datetime(2019, 7, 2, 7, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5956031233275118, DEWP=0.5062258455309152, SLP=-0.5200751586544888, ALT=-0.5060158468808076, STP=-0.5125051372204353, PCP01=-0.17811912685987313, count=52), Row(starttime=datetime.datetime(2019, 7, 2, 8, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.7602102625155688, DEWP=0.5062258455309152, SLP=-0.5977062419217477, ALT=-0.6114899227490619, STP=-0.6063805822756951, PCP01=-0.17811912685987313, count=102), Row(starttime=datetime.datetime(2019, 7, 2, 9, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=0.5062258455309152, SLP=-0.6753373251890066, ALT=-0.6642269606831984, STP=-0.668964212312529, PCP01=-0.17811912685987313, count=66), Row(starttime=datetime.datetime(2019, 7, 2, 10, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.43342055448116873, SLP=-0.7219159751493724, ALT=-0.7169639986173161, STP=-0.7159019348401501, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 7, 2, 11, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.5062258455309152, SLP=-0.7219159751493724, ALT=-0.7169639986173161, STP=-0.7159019348401501, PCP01=-0.17811912685987313, count=14), Row(starttime=datetime.datetime(2019, 7, 2, 12, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.5062258455309152, SLP=-0.7995470584166313, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=17), Row(starttime=datetime.datetime(2019, 7, 2, 13, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.5062258455309152, SLP=-0.8305994917235242, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=15), Row(starttime=datetime.datetime(2019, 7, 2, 14, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.7246417186801546, SLP=-0.8305994917235242, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=19), Row(starttime=datetime.datetime(2019, 7, 2, 15, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.7246417186801546, SLP=-0.954809224951149, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=-0.17811912685987313, count=25), Row(starttime=datetime.datetime(2019, 7, 2, 16, 0), SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.418638819267797, DEWP=1.0158628828791405, SLP=-1.0479665248718455, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=44), Row(starttime=datetime.datetime(2019, 7, 2, 17, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=1.0158628828791405, SLP=-1.0790189581787561, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=112), Row(starttime=datetime.datetime(2019, 7, 2, 18, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.418638819267797, DEWP=0.9430575918293941, SLP=-1.0945451748322113, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=122), Row(starttime=datetime.datetime(2019, 7, 2, 19, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.418638819267797, DEWP=0.9430575918293941, SLP=-1.0634927415253008, ALT=-1.0333862262220976, STP=-1.0288200850243374, PCP01=-0.17811912685987313, count=28), Row(starttime=datetime.datetime(2019, 7, 2, 20, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.8702523007796475, SLP=-1.0790189581787561, ALT=-1.0861232641562153, STP=-1.0914037150611713, PCP01=-0.17811912685987313, count=14), Row(starttime=datetime.datetime(2019, 7, 2, 21, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.089424540891683, DEWP=0.9430575918293941, SLP=-0.9703354416045866, ALT=-0.9806491882879611, STP=-0.9818823624967165, PCP01=-0.17811912685987313, count=13), Row(starttime=datetime.datetime(2019, 7, 2, 22, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.6779066929215403, DEWP=1.2342787560283799, SLP=-0.9237567916442384, ALT=-0.9279121503538433, STP=-0.9349446399690776, PCP01=1.0288804720700693, count=4), Row(starttime=datetime.datetime(2019, 7, 2, 23, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.9248174017036258, DEWP=1.088668173928887, SLP=-0.8616519250304349, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=5), Row(starttime=datetime.datetime(2019, 7, 3, 0, 0), SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=1.088668173928887, SLP=-0.8305994917235242, ALT=-0.8224380744855891, STP=-0.8254232874046228, PCP01=-0.17811912685987313, count=1), Row(starttime=datetime.datetime(2019, 7, 3, 1, 0), SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=0, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.0071209712976543, DEWP=1.0158628828791405, SLP=-0.8616519250304349, ALT=-0.8751751124197069, STP=-0.8723610099322437, PCP01=-0.17811912685987313, count=1)]

from pyspark.sql.types import *
from pyspark.sql import *
from pyspark.sql.types import StructType, StructField, LongType, StringType
from pyspark.sql import Row
from pyspark.sql import Column
import pandas as pd
import numpy as np

train_df=spark.createDataFrame(train_list)
test_df=spark.createDataFrame(test_list)
realtime_df=spark.createDataFrame(realtime_list)
print(train_df.count(),test_df.count(),realtime_df.count())

train_df.cache()
test_df.cache()
realtime_df.cache()

1464 720 740

DataFrame[starttime: timestamp, SPD: double, GUS: double, CLR: bigint, SCT: bigint, BKN: bigint, OVC: bigint, OBS: bigint, POB: bigint, VSB: double, TEMP: double, DEWP: double, SLP: double, ALT: double, STP: double, PCP01: double, count: bigint]

#from pyspark.sql.functions import hour, minute, second
#train_df=train_df.withColumn('starttime', hour(train_df.starttime))
#hour_df=train_df.select(hour('starttime').alias('hour'))
#print(type(hour_df))
#hour_df.show(5)
train_df=train_df.drop("starttime")
test_df=test_df.drop("starttime")
realtime_df=realtime_df.drop("starttime")
train_df.show()
test_df.show()
realtime_df.show()

+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+
|                SPD|                GUS|CLR|SCT|BKN|OVC|OBS|POB|               VSB|               TEMP|               DEWP|                 SLP|                 ALT|                 STP|               PCP01|count|
+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+
| 0.4111950573077732| 3.1687885314639845|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.202718242869458|-2.4059857964589435|-0.00771000909058...|-0.03138250547365...|-0.02748200443494122|-0.17811912685987313|    5|
| 2.8743640151934144| 3.3341854908713513|  0|  1|  0|  0|  0|  0|0.4638614904960641| -2.367325382057515|-2.5515963785584366| 0.10097350748358193| 0.07409157039460014| 0.06639344062031857|-0.17811912685987313|    0|
| 1.9506756559862992|  2.672597653241884|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.531932521245572|-2.6244016696081833|  0.1630783740973855| 0.12682860832873663|  0.1289770706571525|-0.17811912685987313|    0|
| 1.9506756559862992|  2.672597653241884|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.6142360908396003|-2.6244016696081833|  0.2562356740180996| 0.23230268419699088|  0.2384984232216252|-0.17811912685987313|    0|
| 1.6427795362505941|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296|  0.4270240572060726|  0.3905137979993629| 0.39495749831371896|-0.17811912685987313|    0|
| 1.3348834165148888| 3.0033915720566178|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.696539660433629|-2.6972069606579296|  0.5822862237405904|  0.5487249118017536|  0.5514165734058126|-0.17811912685987313|    3|
| 1.3348834165148888| 3.6649794096860853|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.7788432300276575| -2.770012251707676|  0.7220221736216527|  0.7069360256041444|  0.7078756484979064|-0.17811912685987313|   16|
| 1.9506756559862992| 3.3341854908713513|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.7788432300276575| -2.770012251707676|  0.9238629901165186|  0.9178841773406529|  0.9112724461176211|-0.17811912685987313|   39|
| 1.3348834165148888| 2.5072006938345175|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.6142360908396003|-2.6972069606579296|  1.0014940733837774|  0.9706212152747894|   0.973856076154455|-0.17811912685987313|   63|
| 1.3348834165148888|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.4496289516515435|-2.6972069606579296|  1.1257038066114022|  1.0760952911430437|  1.0833774287189277|-0.17811912685987313|   38|
| 1.0269872967791835| 3.6649794096860853|  1|  0|  0|  0|  0|  0|0.4638614904960641|-2.2850218124634862|-2.6972069606579296|  1.2499135398390093|  1.2343064049454158|  1.2398365038110215|-0.17811912685987313|   18|
| 1.3348834165148888| 3.6649794096860853|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.202718242869458|-2.6972069606579296|  1.3120184064528129|   1.287043442879552|  1.2867742263386424|-0.17811912685987313|    7|
| 2.2585717757220043| 3.1687885314639845|  1|  0|  0|  0|  0|  0|0.4638614904960641| -2.038111103681401|-2.8428175427574227|  1.3585970564131788|    1.33978048081367|  1.3337119488662812|-0.17811912685987313|   10|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.8735039644933438| -2.915622833807169|  1.3585970564131788|    1.33978048081367|  1.3337119488662812|-0.17811912685987313|   16|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-2.9884281248569153|  1.3741232730666164|    1.33978048081367|  1.3337119488662812|-0.17811912685987313|   17|
| 1.9506756559862992| 3.0033915720566178|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-3.1340387069564084|  1.3896494897200717|  1.3925175187478065|  1.3962955789031153|-0.17811912685987313|   22|
| 1.9506756559862992|  2.672597653241884|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-3.2068439980061547|  1.4362281396804375|  1.4452545566819428|  1.4432333014307361|-0.17811912685987313|   32|
| 2.2585717757220043| 3.1687885314639845|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.7912003948993154|-3.2796492890559015|  1.5914903062149552|  1.5507286325501972|  1.5527546539952088|-0.17811912685987313|   86|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.8735039644933438|-3.2068439980061547|  1.7157000394425623|   1.708939746352569|  1.7092137290873026|-0.17811912685987313|  133|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-1.9558075340873724|-3.2796492890559015|  1.8399097726701694|  1.8144138222208235|  1.8187350816517576|-0.17811912685987313|   40|
+-------------------+-------------------+---+---+---+---+---+---+------------------+-------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+
only showing top 20 rows

+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-----+
|                SPD|                GUS|CLR|SCT|BKN|OVC|OBS|POB|               VSB|                TEMP|                DEWP|                SLP|                ALT|                STP|               PCP01|count|
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-----+
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.3486924145454262| 0.28780997238167577|-0.7995470584166313|-0.7697010365514526|-0.7628396573677888|-0.17811912685987313|    5|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.2663888449513977| 0.28780997238167577|-0.8305994917235242|-0.8224380744855891|-0.8254232874046228|-0.17811912685987313|    8|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.10178170576334064| 0.36061526343142225|-0.9392830082976936|-0.9279121503538433|-0.9349446399690776|-0.17811912685987313|    2|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-0.06282543342471639| 0.36061526343142225|-0.8771781416838901|-0.8751751124197069|-0.8723610099322437|-0.17811912685987313|    1|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-0.22743257261277341| 0.43342055448116873|-0.7374421918028101|-0.7169639986173161|-0.7159019348401501|-0.17811912685987313|    0|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-0.22743257261277341| 0.28780997238167577|-0.6287586752286582|-0.6114899227490619|-0.6063805822756951|-0.17811912685987313|    1|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-0.22743257261277341| 0.28780997238167577|-0.5511275919613994|-0.5587528848149441|-0.5594428597480563|-0.17811912685987313|    5|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.01947813616931213| 0.06939409923243636|-0.5821800252682924|-0.6114899227490619|-0.6063805822756951|-0.17811912685987313|    1|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.10178170576334064| 0.06939409923243636|-0.4890227253475959|-0.5060158468808076|-0.5125051372204353|-0.17811912685987313|   10|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.2663888449513977|-0.00341119181731...|-0.5200751586544888|-0.5587528848149441|-0.5594428597480563|-0.17811912685987313|   11|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.5132995537334832| 0.06939409923243636|-0.5356013753079442|-0.5587528848149441|-0.5594428597480563|-0.17811912685987313|   13|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.6779066929215403| 0.14219939028218284|-0.5511275919613994|-0.5587528848149441|-0.5594428597480563|-0.17811912685987313|   20|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.7602102625155688| 0.14219939028218284|-0.6753373251890066|-0.6642269606831984| -0.668964212312529|-0.17811912685987313|   22|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.0071209712976543|-0.00341119181731...|-0.8150732750700689|-0.8224380744855891|-0.8254232874046228|-0.17811912685987313|   20|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.9248174017036258| 0.14219939028218284|-0.8771781416838901|-0.8751751124197069|-0.8723610099322437|-0.17811912685987313|   28|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.8425138321095973|  0.2150046813319293| -0.954809224951149|-0.9806491882879611|-0.9818823624967165|-0.17811912685987313|   19|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.5132995537334832| 0.14219939028218284|-0.8927043583373278|-0.9279121503538433|-0.9349446399690776|-0.17811912685987313|    9|
|-0.2045971821636372|-0.3045476160907193|  0|  1|  0|  0|  0|  0|0.4638614904960641|  0.4309959841394547|  0.2150046813319293|-0.9392830082976936|-0.9279121503538433|-0.9349446399690776|-0.17811912685987313|   21|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.2663888449513977| 0.43342055448116873| -0.954809224951149|-0.9806491882879611|-0.9818823624967165|-0.17811912685987313|   17|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.2663888449513977| 0.36061526343142225|-0.9082305749907831|-0.9279121503538433|-0.9349446399690776|-0.17811912685987313|   22|
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+--------------------+-------------------+-------------------+-------------------+--------------------+-----+
only showing top 20 rows

+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+
|                SPD|                GUS|CLR|SCT|BKN|OVC|OBS|POB|               VSB|                TEMP|               DEWP|                 SLP|                 ALT|                 STP|               PCP01|count|
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.18408527535736915|0.06939409923243636|  -0.411391642080337| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313|    2|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.10178170576334064|0.14219939028218284| -0.3492867754665158|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313|    1|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.01947813616931213|0.14219939028218284| -0.2871819088527123| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313|    1|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.01947813616931213|0.06939409923243636|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|    0|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-0.06282543342471639|0.06939409923243636|-0.24060325889236406|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|    1|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| -0.1451290030187449|0.14219939028218284|-0.20955082558545346|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|    8|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-0.06282543342471639| 0.2150046813319293|-0.19402460893199816|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313|   16|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.10178170576334064| 0.2150046813319293|-0.14744595897164992|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313|   52|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.3486924145454262|0.36061526343142225|-0.13191974231819462|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313|   69|
| 1.9506756559862992|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.5132995537334832|0.28780997238167577|-0.08534109235784636|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313|   54|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.6779066929215403|0.36061526343142225|-0.10086730901130166|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313|   21|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.1717281104857113|0.28780997238167577|-0.16297217562510521|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313|   15|
| 1.3348834165148888|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|   1.089424540891683| 0.2150046813319293| -0.1784983922785605|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313|   15|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.3363352496737684| 0.2150046813319293|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|   17|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|    1.25403168007974| 0.2150046813319293| -0.3337605588130782|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313|   12|
|-0.2045971821636372|-0.3045476160907193|  0|  1|  0|  0|  0|  0|0.4638614904960641|  1.3363352496737684|0.14219939028218284| -0.3182343421596229| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313|   27|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.3363352496737684|0.06939409923243636|-0.39586542542688175| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313|   47|
| 1.0269872967791835|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|    1.25403168007974|0.06939409923243636|-0.44244407538722996|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313|  165|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.1717281104857113|0.28780997238167577|-0.47349650869414056|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313|  128|
| 1.0269872967791835|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.1717281104857113|0.06939409923243636| -0.4579702920406853|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313|   36|
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+
only showing top 20 rows

#a=train_df
#from pyspark.sql.functions import hour, minute, second
#hour_df=a.select(hour('starttime').alias('hour'))
#hour_df.show()
#from pyspark.sql.functions import lit

#a=a.withColumn('hour',lit(None))
#a=a.withColumn("hour", lit(1))
#a.show()
#print(type(a.count()))

# 1.build the pipeline of desision tree
from pyspark.ml import Pipeline
from pyspark.ml.feature import VectorAssembler, VectorIndexer, StringIndexer
from pyspark.ml.regression import DecisionTreeRegressor

features_column = train_df.columns[:-1]
print(features_column)
vector = VectorAssembler(inputCols=features_column, outputCol="va_features")
vIndexer = VectorIndexer(inputCol="va_features", outputCol="features", maxCategories=10)
dt = DecisionTreeRegressor(featuresCol="features", labelCol="count", seed=1024)

dt_pipeline = Pipeline(stages=[vector, vIndexer, dt])
dt_pipeline.getStages()

['SPD', 'GUS', 'CLR', 'SCT', 'BKN', 'OVC', 'OBS', 'POB', 'VSB', 'TEMP', 'DEWP', 'SLP', 'ALT', 'STP', 'PCP01']

[VectorAssembler_76be220d6dbb,
 VectorIndexer_d6c3a8f78212,
 DecisionTreeRegressor_e584d684e8e9]

dtModel = dt_pipeline.fit(train_df)
prediction = dtModel.transform(test_df)
prediction.columns

['SPD',
 'GUS',
 'CLR',
 'SCT',
 'BKN',
 'OVC',
 'OBS',
 'POB',
 'VSB',
 'TEMP',
 'DEWP',
 'SLP',
 'ALT',
 'STP',
 'PCP01',
 'count',
 'va_features',
 'features',
 'prediction']

prediction.select("count", "va_features","features","prediction").show()

+-----+--------------------+--------------------+------------------+
|count|         va_features|            features|        prediction|
+-----+--------------------+--------------------+------------------+
|    5|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737|
|    8|[0.41119505730777...|[0.41119505730777...|15.254098360655737|
|    2|[0.41119505730777...|[0.41119505730777...|15.254098360655737|
|    1|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737|
|    0|[-1.1282855413707...|[-1.1282855413707...| 15.72347266881029|
|    1|[0.41119505730777...|[0.41119505730777...| 15.72347266881029|
|    5|[-1.1282855413707...|[-1.1282855413707...| 15.72347266881029|
|    1|[-1.1282855413707...|[-1.1282855413707...|              73.0|
|   10|[0.71909117704347...|[0.71909117704347...|              73.0|
|   11|[-1.1282855413707...|[-1.1282855413707...|23.631067961165048|
|   13|[0.71909117704347...|[0.71909117704347...|              73.0|
|   20|[-1.1282855413707...|[-1.1282855413707...|15.254098360655737|
|   22|[0.71909117704347...|[0.71909117704347...|15.254098360655737|
|   20|[-1.1282855413707...|[-1.1282855413707...|23.631067961165048|
|   28|[0.41119505730777...|[0.41119505730777...|15.254098360655737|
|   19|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737|
|    9|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737|
|   21|[-0.2045971821636...|[-0.2045971821636...|28.137931034482758|
|   17|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737|
|   22|[0.71909117704347...|[0.71909117704347...|15.254098360655737|
+-----+--------------------+--------------------+------------------+
only showing top 20 rows

from pyspark.ml.evaluation import RegressionEvaluator
evaluator = RegressionEvaluator(labelCol="count", predictionCol="prediction", metricName="rmse")
rmse = evaluator.evaluate(prediction)
rmse

36.87194499840543

#realtime streaming
realtime_list=realtime_df.collect()
newtrain_list=train_df.collect()

july_prediction1_list=[]
for i in range(int(realtime_df.count()/12)):
    print(i+1) 
    realtimepart_list=realtime_list[i*12:(i*12+12)] 
    realtimepart_df = spark.createDataFrame(realtimepart_list)
    prediction = dtModel.transform(realtimepart_df)
    prediction.select("count", "prediction").show()
    rmse = evaluator.evaluate(prediction)
    print(rmse)
    july_prediction1_list=july_prediction1_list+prediction.collect()
    
    newtrain_list=newtrain_list + realtimepart_list
    newtrain_df = spark.createDataFrame(newtrain_list)
    dtModel = dt_pipeline.fit(newtrain_df)
    
print(type(july_prediction1_list)) 
print(len(july_prediction1_list))
print(july_prediction1_list[0:20])
july_prediction1_df = spark.createDataFrame(july_prediction1_list)
july_prediction1_df.show()

1
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2|              73.0|
|    1|15.254098360655737|
|    1|15.254098360655737|
|    0|              73.0|
|    1|              73.0|
|    8| 15.72347266881029|
|   16|15.254098360655737|
|   52|15.254098360655737|
|   69|15.254098360655737|
|   54|15.254098360655737|
|   21|15.254098360655737|
|   15|15.254098360655737|
+-----+------------------+

42.616287100960605
2
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   15| 18.88695652173913|
|   17| 18.88695652173913|
|   12| 18.88695652173913|
|   27|36.416666666666664|
|   47| 18.88695652173913|
|  165| 18.88695652173913|
|  128| 18.88695652173913|
|   36| 18.88695652173913|
|   18| 18.88695652173913|
|    7| 18.88695652173913|
|   11| 18.88695652173913|
|    7| 18.88695652173913|
+-----+------------------+

53.88122761003702
3
+-----+-----------------+
|count|       prediction|
+-----+-----------------+
|    2|19.72072072072072|
|    3|19.72072072072072|
|    1|19.72072072072072|
|    2|19.72072072072072|
|    0|           33.675|
|    5|19.72072072072072|
|   23|19.72072072072072|
|   52|19.72072072072072|
|  102|19.72072072072072|
|   66|19.72072072072072|
|   17|19.72072072072072|
|   14|19.72072072072072|
+-----+-----------------+

32.42078424727159
4
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   17|20.295081967213115|
|   15|20.295081967213115|
|   19|20.295081967213115|
|   25|20.295081967213115|
|   44|20.295081967213115|
|  112|20.295081967213115|
|  122|20.295081967213115|
|   28|20.295081967213115|
|   14|20.295081967213115|
|   13|20.295081967213115|
|    4|20.295081967213115|
|    5|20.295081967213115|
+-----+------------------+

40.85543180949201
5
+-----+----------+
|count|prediction|
+-----+----------+
|    1|    19.848|
|    1|    19.848|
|    0|    19.848|
|    1|    19.848|
|    0|    19.848|
|    5|    19.848|
|   17|    19.848|
|   33|    19.848|
|   58|    19.848|
|   39|    19.848|
|   17|    19.848|
|   13|    19.848|
+-----+----------+

18.55747209796277
6
+-----+----------+
|count|prediction|
+-----+----------+
|   17|      18.0|
|   37|      18.0|
|   44|      18.0|
|   78|      18.0|
|   67|      18.0|
|   93|      28.0|
|   36|      18.0|
|   20|      18.0|
|   15|      18.0|
|    7|      18.0|
|    7|      18.0|
|    6|      18.0|
+-----+----------+

31.605642956071414
7
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3| 18.10569105691057|
|    0| 18.10569105691057|
|    0| 18.10569105691057|
|    0| 18.10569105691057|
|    0| 18.10569105691057|
|    0| 18.10569105691057|
|    2| 18.10569105691057|
|    3| 18.10569105691057|
|    2|31.967741935483872|
|    5| 46.23529411764706|
|    8|31.967741935483872|
|   12|31.967741935483872|
+-----+------------------+

22.22314060734218
8
+-----+-----------------+
|count|       prediction|
+-----+-----------------+
|   14|10.51923076923077|
|    9|10.51923076923077|
|   18|10.51923076923077|
|   19|10.51923076923077|
|   19|10.51923076923077|
|   16|10.51923076923077|
|   14|10.51923076923077|
|   12|10.51923076923077|
|   11|10.51923076923077|
|    3|10.51923076923077|
|   15|10.51923076923077|
|    7|10.51923076923077|
+-----+-----------------+

5.383711004637931
9
+-----+----------+
|count|prediction|
+-----+----------+
|    1|      11.0|
|    1|      11.0|
|    1|      11.0|
|    0|      11.0|
|    0|      11.0|
|    1|      11.0|
|    6|      11.0|
|   15|      11.0|
|   26|      11.0|
|   16|      11.0|
|   10|      11.0|
|    8|      11.0|
+-----+----------+

8.864724097981467
10
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   15|18.253658536585366|
|   15|18.253658536585366|
|   16|18.253658536585366|
|   23|18.253658536585366|
|   24|18.253658536585366|
|   49|18.253658536585366|
|   21|18.253658536585366|
|   18|18.253658536585366|
|   15|18.253658536585366|
|    7|18.253658536585366|
|    7|18.253658536585366|
|   11|18.253658536585366|
+-----+------------------+

10.61145421416695
11
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    7|18.262672811059907|
|    2|18.262672811059907|
|    1|18.262672811059907|
|    1|18.262672811059907|
|    0|18.262672811059907|
|    2|18.262672811059907|
|    2|18.262672811059907|
|    4|18.262672811059907|
|    4|18.262672811059907|
|    3|18.262672811059907|
|    3|18.262672811059907|
|   15|18.262672811059907|
+-----+------------------+

15.091905697737033
12
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   16| 17.29004329004329|
|   14| 49.44444444444444|
|   17| 49.44444444444444|
|   12| 49.44444444444444|
|   17| 49.44444444444444|
|   12| 49.44444444444444|
|   18| 49.44444444444444|
|   14|30.920245398773005|
|   13|30.920245398773005|
|   13|30.920245398773005|
|    8|30.920245398773005|
|    8|30.920245398773005|
+-----+------------------+

27.591398302049143
13
+-----+-----------------+
|count|       prediction|
+-----+-----------------+
|    7|18.00796812749004|
|    3|18.00796812749004|
|    1|18.00796812749004|
|    0|18.00796812749004|
|    0|18.00796812749004|
|    0|18.00796812749004|
|    1|18.00796812749004|
|    3|18.00796812749004|
|    4|18.00796812749004|
|    5|18.00796812749004|
|    6|18.00796812749004|
|   10|18.00796812749004|
+-----+-----------------+

14.991123881509472
14
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   13|17.338403041825096|
|   17|17.338403041825096|
|   19|17.338403041825096|
|   10|17.338403041825096|
|   16|17.338403041825096|
|   20|17.338403041825096|
|   16|17.338403041825096|
|   12|17.338403041825096|
|    6|17.338403041825096|
|    5|17.338403041825096|
|    8|17.338403041825096|
|    6|17.338403041825096|
+-----+------------------+

7.137199147827578
15
+-----+----------+
|count|prediction|
+-----+----------+
|    0|     17.12|
|    0|     17.12|
|    0|     17.12|
|    0|     17.12|
|    0|     17.12|
|    3|     17.12|
|   18|     17.12|
|   31|     17.12|
|   51|     17.12|
|   32|     17.12|
|   30|     17.12|
|    3|     17.12|
+-----+----------+

17.303209721513134
16
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    9|16.989547038327526|
|   14|16.989547038327526|
|   18|16.989547038327526|
|   20|16.989547038327526|
|   52|16.989547038327526|
|  165|16.989547038327526|
|  118|16.989547038327526|
|   38|16.989547038327526|
|   19|16.989547038327526|
|   12|16.989547038327526|
|    7|16.989547038327526|
|    3|16.989547038327526|
+-----+------------------+

53.37342254999157
17
+-----+----------+
|count|prediction|
+-----+----------+
|    0|      22.5|
|    1|      22.5|
|    0|      22.5|
|    0|      22.5|
|    0|      22.5|
|    7|      22.5|
|   20|      22.5|
|   48|      22.5|
|  106|      22.5|
|   68|      22.5|
|   13|      22.5|
|   20|      22.5|
+-----+----------+

32.30454044454639
18
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   14|21.949868073878626|
|   13|21.949868073878626|
|   16|21.949868073878626|
|   21|21.949868073878626|
|   51|21.949868073878626|
|  163|21.949868073878626|
|  108|21.949868073878626|
|   33|21.949868073878626|
|   21|21.949868073878626|
|   16|21.949868073878626|
|    5|21.949868073878626|
|    4|21.949868073878626|
+-----+------------------+

49.23682835102618
19
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3|22.465473145780052|
|    2|22.465473145780052|
|    0|22.465473145780052|
|    1|22.465473145780052|
|    0|22.465473145780052|
|    7|22.465473145780052|
|   22|22.465473145780052|
|   59|22.465473145780052|
|  105|22.465473145780052|
|  100|22.465473145780052|
|   38|22.465473145780052|
|   14|22.465473145780052|
+-----+------------------+

37.614943457385365
20
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   22|22.667493796526056|
|   21|22.667493796526056|
|   20|22.667493796526056|
|   19|22.667493796526056|
|   41|22.667493796526056|
|  147|22.667493796526056|
|  123|22.667493796526056|
|   33|22.667493796526056|
|   20|22.667493796526056|
|   16|22.667493796526056|
|    4|22.667493796526056|
|    3|22.667493796526056|
+-----+------------------+

47.23934987472937
21
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3|23.142168674698794|
|    3|23.142168674698794|
|    0|23.142168674698794|
|    1|23.142168674698794|
|    1|23.142168674698794|
|    6|23.142168674698794|
|   16|23.142168674698794|
|   56|23.142168674698794|
|   83|23.142168674698794|
|   62|23.142168674698794|
|   18|23.142168674698794|
|   10|23.142168674698794|
+-----+------------------+

27.45310589308695
22
+-----+-----------------+
|count|       prediction|
+-----+-----------------+
|   14|8.918367346938776|
|   12|8.918367346938776|
|   18|8.918367346938776|
|    4|             56.0|
|   30|              9.5|
|   79|              9.5|
|   76|             2.25|
|    7|              9.5|
|    5|             56.0|
|    3|              9.5|
|    3|              9.5|
|    1|60.61290322580645|
+-----+-----------------+

40.57586128977382
23
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    5|27.333333333333332|
|    1|27.333333333333332|
|    1|27.333333333333332|
|    0|27.333333333333332|
|    0|27.333333333333332|
|    3|27.333333333333332|
|   21|27.333333333333332|
|   52|27.333333333333332|
|   75| 59.92857142857143|
|   67| 59.92857142857143|
|   20| 59.92857142857143|
|   11| 59.92857142857143|
+-----+------------------+

27.212136592501324
24
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   19|55.970588235294116|
|   16|55.970588235294116|
|   25|55.970588235294116|
|   31|55.970588235294116|
|   52|55.970588235294116|
|  128|55.970588235294116|
|   67|23.385390428211586|
|   35|23.385390428211586|
|   21|23.385390428211586|
|    8|23.385390428211586|
|   11|23.385390428211586|
|    8|23.385390428211586|
+-----+------------------+

32.17248234816456
25
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3|26.408256880733944|
|    2| 28.63362068965517|
|    3|26.408256880733944|
|    2|26.408256880733944|
|    0|26.408256880733944|
|    0|26.408256880733944|
|    3|26.408256880733944|
|    6|26.408256880733944|
|    6|26.408256880733944|
|    6| 28.63362068965517|
|    5|26.408256880733944|
|   16|26.408256880733944|
+-----+------------------+

22.832112338121764
26
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    9|29.141025641025642|
|   19|29.141025641025642|
|   18|29.141025641025642|
|   23|29.141025641025642|
|   21|29.141025641025642|
|   23|29.141025641025642|
|   25|29.141025641025642|
|   18|29.141025641025642|
|   17|29.141025641025642|
|   14|29.141025641025642|
|   14|29.141025641025642|
|   13|16.632373113854594|
+-----+------------------+

11.308901200182975
27
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    4|19.624719101123596|
|    3|19.624719101123596|
|    1|19.624719101123596|
|    0|19.624719101123596|
|    0|19.624719101123596|
|    3|19.624719101123596|
|    4|19.624719101123596|
|    4|19.624719101123596|
|    5|19.624719101123596|
|    3|19.624719101123596|
|   13| 32.61363636363637|
|   13| 32.61363636363637|
+-----+------------------+

17.468902439595574
28
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   15|              29.5|
|   18|              29.5|
|   33|              29.5|
|   20|              29.5|
|   26|              29.5|
|   16|              29.0|
|    8|             164.0|
|    9|             164.0|
|    7|             164.0|
|    3|             164.0|
|    4|15.958579881656805|
|    2|28.732217573221757|
+-----+------------------+

91.47625788930519
29
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    4|17.392290249433106|
|    1|17.392290249433106|
|    0|17.392290249433106|
|    0|17.392290249433106|
|    0|17.392290249433106|
|    7|17.392290249433106|
|   28|17.392290249433106|
|   68|17.392290249433106|
|   92|17.392290249433106|
|   66|17.392290249433106|
|   18| 24.27077747989276|
|   10|17.392290249433106|
+-----+------------------+

31.833142836045685
30
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   11|17.579646017699115|
|   18|17.579646017699115|
|   14|17.579646017699115|
|   22|17.579646017699115|
|   54|17.579646017699115|
|  158|17.579646017699115|
|  109|17.579646017699115|
|   43|17.579646017699115|
|   21|17.579646017699115|
|    8|17.579646017699115|
|    8|17.579646017699115|
|    4|17.579646017699115|
+-----+------------------+

50.41796983934571
31
+-----+-----------------+
|count|       prediction|
+-----+-----------------+
|    1|20.66176470588235|
|    5|20.66176470588235|
|    1|20.66176470588235|
|    0|20.66176470588235|
|    0|20.66176470588235|
|    3|20.66176470588235|
|   25|20.66176470588235|
|   51|20.66176470588235|
|   92|20.66176470588235|
|   76|20.66176470588235|
|   23|           20.625|
|   20|            14.25|
+-----+-----------------+

30.70506454157064
32
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   16|14.523809523809524|
|   10|14.523809523809524|
|   22| 20.88888888888889|
|   19| 63.05555555555556|
|   53| 63.05555555555556|
|  161| 63.05555555555556|
|  105| 63.05555555555556|
|   37| 63.05555555555556|
|   21| 63.05555555555556|
|   20| 63.05555555555556|
|    5| 63.05555555555556|
|    3| 63.05555555555556|
+-----+------------------+

45.365915275584015
33
+-----+-----------------+
|count|       prediction|
+-----+-----------------+
|    2|              6.5|
|    0|              6.5|
|    1|19.06644518272425|
|    2|19.06644518272425|
|    1|19.06644518272425|
|    8|19.06644518272425|
|   19|19.06644518272425|
|   63|              6.5|
|   83|              6.5|
|   62|            63.44|
|   25|17.32608695652174|
|    8|17.32608695652174|
+-----+-----------------+

29.327838733757886
34
+-----+-----------------+
|count|       prediction|
+-----+-----------------+
|   13|16.47826086956522|
|   15|16.47826086956522|
|   15|36.23529411764706|
|   16|36.23529411764706|
|   39|36.23529411764706|
|  145|36.23529411764706|
|  110|36.23529411764706|
|   41|36.23529411764706|
|    7|             8.12|
|    5|             65.0|
|    3|             8.12|
|    3|             8.12|
+-----+-----------------+

42.651407458714964
35
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2| 19.93632075471698|
|    1| 19.93632075471698|
|    0| 7.888888888888889|
|    0| 7.888888888888889|
|    0|27.694560669456067|
|    3| 19.93632075471698|
|   14| 19.93632075471698|
|   47| 19.93632075471698|
|   43| 19.93632075471698|
|   28| 7.888888888888889|
|    7| 7.888888888888889|
|    9|27.694560669456067|
+-----+------------------+

18.05764400480881
36
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    9|20.010526315789473|
|   12|19.913953488372094|
|   30|20.010526315789473|
|   19|19.913953488372094|
|   34|20.010526315789473|
|  155|20.010526315789473|
|   93|20.010526315789473|
|   37|20.010526315789473|
|   15|20.010526315789473|
|    6|20.010526315789473|
|    5|20.010526315789473|
|    5|20.010526315789473|
+-----+------------------+

45.6348625244084
37
+-----+-----------------+
|count|       prediction|
+-----+-----------------+
|    2|19.89351851851852|
|    5|19.89351851851852|
|    0|19.89351851851852|
|    0|19.89351851851852|
|    0|19.89351851851852|
|    6|19.89351851851852|
|   14|19.89351851851852|
|   40|19.89351851851852|
|   58|19.89351851851852|
|   59|27.95617529880478|
|   15|19.89351851851852|
|   17|19.89351851851852|
+-----+-----------------+

20.019630172193395
38
+-----+-----------------+
|count|       prediction|
+-----+-----------------+
|    7|            13.75|
|   24|             25.9|
|   31|60.93333333333333|
|   43|60.93333333333333|
|   72|60.93333333333333|
|   91|60.93333333333333|
|   51|             25.9|
|   30|             25.9|
|   17|             25.9|
|    8|             25.9|
|    2|60.93333333333333|
|    8|60.93333333333333|
+-----+-----------------+

28.30602482982495
39
+-----+-----------------+
|count|       prediction|
+-----+-----------------+
|    6|38.72549019607843|
|    1|38.72549019607843|
|    2|38.72549019607843|
|    0|38.72549019607843|
|    1|38.72549019607843|
|    1|38.72549019607843|
|    1|38.72549019607843|
|    2|38.72549019607843|
|    4|38.72549019607843|
|   10|38.72549019607843|
|   12|38.72549019607843|
|   11|38.72549019607843|
+-----+-----------------+

34.73106569427291
40
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   11|60.733333333333334|
|   12|60.733333333333334|
|   10|60.733333333333334|
|   15|60.733333333333334|
|   10|60.733333333333334|
|   17|60.733333333333334|
|   10|60.733333333333334|
|   13|60.733333333333334|
|   11|60.733333333333334|
|    9|60.733333333333334|
|    4|60.733333333333334|
|    6|60.733333333333334|
+-----+------------------+

50.18027500921054
41
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1|15.631578947368421|
|    2|15.631578947368421|
|    0|15.631578947368421|
|    0|15.631578947368421|
|    0|15.631578947368421|
|    1|26.931506849315067|
|    1|15.631578947368421|
|    4|15.631578947368421|
|    5|15.631578947368421|
|    3|15.631578947368421|
|    5|15.631578947368421|
|    7|15.631578947368421|
+-----+------------------+

14.761671909287497
42
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   16|18.884318766066837|
|   16|18.884318766066837|
|   16|18.884318766066837|
|   15|18.884318766066837|
|   11|52.674418604651166|
|   15|18.884318766066837|
|   16|18.884318766066837|
|    6|18.884318766066837|
|    7|18.884318766066837|
|    7|18.884318766066837|
|    3|18.884318766066837|
|    2|18.884318766066837|
+-----+------------------+

15.237239873509608
43
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3|18.661654135338345|
|    2|18.661654135338345|
|    3|18.661654135338345|
|    0|18.661654135338345|
|    0|18.661654135338345|
|    4|18.661654135338345|
|   23|18.661654135338345|
|   53|18.661654135338345|
|   70|18.661654135338345|
|   55|18.661654135338345|
|   16|18.661654135338345|
|    9|18.661654135338345|
+-----+------------------+

24.038407899967844
44
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    9|25.674107142857142|
|   14|25.674107142857142|
|   12|50.680851063829785|
|   14|              17.2|
|   27|               5.4|
|   89|50.680851063829785|
|   15|              17.2|
|    4|              79.0|
|    9|20.848484848484848|
|   13|50.680851063829785|
|    8|50.680851063829785|
|    6|  19.0813704496788|
+-----+------------------+

32.962764941777884
45
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    0|             16.75|
|    1| 52.91111111111111|
|    0|              6.75|
|    1|              6.75|
|    0|              6.75|
|    1|              79.0|
|    0|              6.75|
|   10|              6.75|
|   77|18.966101694915253|
|   63|18.966101694915253|
|   15|18.966101694915253|
|   17|18.966101694915253|
+-----+------------------+

34.840292200921084
46
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   16|19.215031315240083|
|   13|19.215031315240083|
|   18|19.215031315240083|
|   22|19.215031315240083|
|   31|19.215031315240083|
|  116|19.215031315240083|
|  138|19.215031315240083|
|   41|19.215031315240083|
|   20|19.215031315240083|
|    8|19.215031315240083|
|   11|19.215031315240083|
|    6|19.215031315240083|
+-----+------------------+

45.2008310984955
47
+-----+-----------------+
|count|       prediction|
+-----+-----------------+
|    1|23.27992277992278|
|    0|              6.5|
|    1|23.27992277992278|
|    1|23.27992277992278|
|    0|23.27992277992278|
|    6|23.27992277992278|
|   26|23.27992277992278|
|   62|23.27992277992278|
|   90|23.27992277992278|
|   93|23.27992277992278|
|   31|23.27992277992278|
|   22|23.27992277992278|
+-----+-----------------+

33.23191773932889
48
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   28|23.425330812854444|
|   26|23.425330812854444|
|   36|23.425330812854444|
|   36|23.425330812854444|
|   54|23.425330812854444|
|  118|23.425330812854444|
|  110|23.425330812854444|
|   57|23.425330812854444|
|   22|23.425330812854444|
|   16|23.425330812854444|
|    6|23.425330812854444|
|    6|23.425330812854444|
+-----+------------------+

40.32148181290375
49
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3|23.868512110726645|
|    6|23.868512110726645|
|    2|23.868512110726645|
|    2|23.868512110726645|
|    2|23.868512110726645|
|    6|23.868512110726645|
|   21|23.868512110726645|
|   54|23.868512110726645|
|   89|23.868512110726645|
|   71|23.868512110726645|
|   24|23.868512110726645|
|   13|23.868512110726645|
+-----+------------------+

28.87692612287684
50
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   13| 25.83783783783784|
|   23| 25.83783783783784|
|   13| 25.83783783783784|
|   30| 25.83783783783784|
|   55| 25.83783783783784|
|  142| 25.83783783783784|
|  112| 25.83783783783784|
|   50|19.400437636761488|
|   27| 25.83783783783784|
|   21|19.400437636761488|
|   22|19.400437636761488|
|   13|19.400437636761488|
+-----+------------------+

43.885229719754875
51
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    8| 26.44868035190616|
|    0| 26.44868035190616|
|    1| 26.44868035190616|
|    1|19.462039045553144|
|    2|19.462039045553144|
|    7|19.462039045553144|
|   23|19.462039045553144|
|   37|19.462039045553144|
|   71|19.462039045553144|
|   67| 26.44868035190616|
|   18| 26.44868035190616|
|   12| 26.44868035190616|
+-----+------------------+

24.814993269455666
52
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   18|26.543408360128616|
|   22|26.543408360128616|
|   37|26.543408360128616|
|   42| 19.51502145922747|
|   80| 19.51502145922747|
|  105|26.543408360128616|
|   65|26.543408360128616|
|   29|26.543408360128616|
|   15|26.543408360128616|
|   23|26.543408360128616|
|   11|26.543408360128616|
|    7|26.543408360128616|
+-----+------------------+

32.63013562600372
53
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    7| 26.75077881619938|
|    2| 26.75077881619938|
|    2| 26.75077881619938|
|    1| 26.75077881619938|
|    0|19.692307692307693|
|    0| 26.75077881619938|
|    5|19.692307692307693|
|    4|19.692307692307693|
|    8|19.692307692307693|
|   12|19.692307692307693|
|   10|19.692307692307693|
|   21|19.692307692307693|
+-----+------------------+

18.56393029829631
54
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   18|19.555555555555557|
|   13|19.555555555555557|
|   13|19.555555555555557|
|   23|19.555555555555557|
|   25|19.555555555555557|
|   21|19.555555555555557|
|   25|19.555555555555557|
|   15|19.555555555555557|
|   11|19.555555555555557|
|   11|19.555555555555557|
|   11|19.555555555555557|
|    3|19.555555555555557|
+-----+------------------+

7.505759105719183
55
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2|19.460416666666667|
|    1|19.460416666666667|
|    2|19.460416666666667|
|    0|19.460416666666667|
|    0|19.460416666666667|
|    1|19.460416666666667|
|    1|19.460416666666667|
|    0|19.460416666666667|
|    4|19.460416666666667|
|    6|19.460416666666667|
|   10|19.460416666666667|
|   10|19.460416666666667|
+-----+------------------+

16.75167396797009
56
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   13|27.472868217054263|
|   14|27.472868217054263|
|   21|27.472868217054263|
|    5|27.472868217054263|
|    0|27.472868217054263|
|    0|27.472868217054263|
|    0|27.472868217054263|
|    0|27.472868217054263|
|    0|27.472868217054263|
|    0|27.472868217054263|
|    0|27.472868217054263|
|    0|19.786941580756015|
+-----+------------------+

23.47470694687588
57
+-----+-----------------+
|count|       prediction|
+-----+-----------------+
|    0|19.77758620689655|
|    0|19.77758620689655|
|    2|19.77758620689655|
|    1|19.77758620689655|
|    0|19.77758620689655|
|    6|19.77758620689655|
|   14|19.77758620689655|
|   44|19.77758620689655|
|   92|19.77758620689655|
|   55|26.54646840148699|
|   23|26.54646840148699|
|   10|26.54646840148699|
+-----+-----------------+

27.331407981264046
58
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   19|21.658620689655173|
|   10|21.658620689655173|
|   16|21.658620689655173|
|   19|21.658620689655173|
|   47|21.658620689655173|
|  148|21.658620689655173|
|  135|21.658620689655173|
|   41|21.658620689655173|
|   25|21.658620689655173|
|    5|21.658620689655173|
|    6|21.658620689655173|
|    3|21.658620689655173|
+-----+------------------+

50.735242343913164
59
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1|24.578034682080926|
|    1|17.508951406649615|
|    1|17.508951406649615|
|    0|17.508951406649615|
|    0|17.508951406649615|
|    8|17.508951406649615|
|   24|17.508951406649615|
|   46|17.508951406649615|
|   94|17.508951406649615|
|   47|24.578034682080926|
|   19|24.578034682080926|
|   19|24.578034682080926|
+-----+------------------+

27.4989229113094
60
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   11|22.278443113772454|
|   11|22.278443113772454|
|   25|22.278443113772454|
|   11|22.278443113772454|
|   42|22.278443113772454|
|  158|22.278443113772454|
|  113|22.278443113772454|
|   34|22.278443113772454|
|   23|22.278443113772454|
|    8|22.278443113772454|
|   10|22.278443113772454|
|    5|22.278443113772454|
+-----+------------------+

48.49380771660473
61
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    0|22.591036414565828|
|    1|22.591036414565828|
|    1|22.591036414565828|
|    3|22.591036414565828|
|    0|22.591036414565828|
|    6|22.591036414565828|
|   24|22.591036414565828|
|   61|22.591036414565828|
|  103|22.591036414565828|
|   64|22.591036414565828|
|   20|22.591036414565828|
|   12|22.591036414565828|
+-----+------------------+

32.130654665517675
<class 'list'>
732
[Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.18408527535736915, DEWP=0.06939409923243636, SLP=-0.411391642080337, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=2, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1841, 0.0694, -0.4114, -0.4005, -0.403, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1841, 0.0694, -0.4114, -0.4005, -0.403, -0.1781]), prediction=73.0), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.14219939028218284, SLP=-0.3492867754665158, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.1422, -0.3493, -0.3478, -0.3404, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.1422, -0.3493, -0.3478, -0.3404, -0.1781]), prediction=15.254098360655737), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.14219939028218284, SLP=-0.2871819088527123, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.1422, -0.2872, -0.2951, -0.2935, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.1422, -0.2872, -0.2951, -0.2935, -0.1781]), prediction=15.254098360655737), Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.06939409923243636, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=0, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.0694, -0.2561, -0.2423, -0.2465, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.0694, -0.2561, -0.2423, -0.2465, -0.1781]), prediction=73.0), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.06939409923243636, SLP=-0.24060325889236406, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.0694, -0.2406, -0.2423, -0.2465, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.0694, -0.2406, -0.2423, -0.2465, -0.1781]), prediction=73.0), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.1451290030187449, DEWP=0.14219939028218284, SLP=-0.20955082558545346, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=8, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.1451, 0.1422, -0.2096, -0.2423, -0.2465, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.1451, 0.1422, -0.2096, -0.2423, -0.2465, -0.1781]), prediction=15.72347266881029), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.2150046813319293, SLP=-0.19402460893199816, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=16, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.215, -0.194, -0.1896, -0.1839, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.215, -0.194, -0.1896, -0.1839, -0.1781]), prediction=15.254098360655737), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.2150046813319293, SLP=-0.14744595897164992, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=52, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.215, -0.1474, -0.1369, -0.137, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.215, -0.1474, -0.1369, -0.137, -0.1781]), prediction=15.254098360655737), Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.3486924145454262, DEWP=0.36061526343142225, SLP=-0.13191974231819462, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=69, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.3487, 0.3606, -0.1319, -0.1369, -0.137, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.3487, 0.3606, -0.1319, -0.1369, -0.137, -0.1781]), prediction=15.254098360655737), Row(SPD=1.9506756559862992, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.28780997238167577, SLP=-0.08534109235784636, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=54, va_features=DenseVector([1.9507, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.5133, 0.2878, -0.0853, -0.0841, -0.0901, -0.1781]), features=DenseVector([1.9507, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.5133, 0.2878, -0.0853, -0.0841, -0.0901, -0.1781]), prediction=15.254098360655737), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.6779066929215403, DEWP=0.36061526343142225, SLP=-0.10086730901130166, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=21, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.6779, 0.3606, -0.1009, -0.0841, -0.0901, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.6779, 0.3606, -0.1009, -0.0841, -0.0901, -0.1781]), prediction=15.254098360655737), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.16297217562510521, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=15, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.163, -0.1369, -0.137, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.163, -0.1369, -0.137, -0.1781]), prediction=15.254098360655737), Row(SPD=1.3348834165148888, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.089424540891683, DEWP=0.2150046813319293, SLP=-0.1784983922785605, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=15, va_features=DenseVector([1.3349, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.0894, 0.215, -0.1785, -0.1896, -0.1839, -0.1781]), features=DenseVector([1.3349, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.0894, 0.215, -0.1785, -0.1896, -0.1839, -0.1781]), prediction=18.88695652173913), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.2150046813319293, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=17, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.215, -0.2561, -0.2423, -0.2465, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.215, -0.2561, -0.2423, -0.2465, -0.1781]), prediction=18.88695652173913), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.2150046813319293, SLP=-0.3337605588130782, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=12, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.215, -0.3338, -0.3478, -0.3404, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.215, -0.3338, -0.3478, -0.3404, -0.1781]), prediction=18.88695652173913), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.14219939028218284, SLP=-0.3182343421596229, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=27, va_features=DenseVector([-0.2046, -0.3045, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.1422, -0.3182, -0.2951, -0.2935, -0.1781]), features=DenseVector([-0.2046, -0.3045, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.1422, -0.3182, -0.2951, -0.2935, -0.1781]), prediction=36.416666666666664), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.06939409923243636, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=47, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.0694, -0.3959, -0.4005, -0.403, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.0694, -0.3959, -0.4005, -0.403, -0.1781]), prediction=18.88695652173913), Row(SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.06939409923243636, SLP=-0.44244407538722996, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=165, va_features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.0694, -0.4424, -0.4533, -0.4499, -0.1781]), features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.0694, -0.4424, -0.4533, -0.4499, -0.1781]), prediction=18.88695652173913), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.47349650869414056, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=128, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.4735, -0.4533, -0.4499, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.4735, -0.4533, -0.4499, -0.1781]), prediction=18.88695652173913), Row(SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.06939409923243636, SLP=-0.4579702920406853, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=36, va_features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.0694, -0.458, -0.4533, -0.4499, -0.1781]), features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.0694, -0.458, -0.4533, -0.4499, -0.1781]), prediction=18.88695652173913)]
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+
|                SPD|                GUS|CLR|SCT|BKN|OVC|OBS|POB|               VSB|                TEMP|               DEWP|                 SLP|                 ALT|                 STP|               PCP01|count|         va_features|            features|        prediction|
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.18408527535736915|0.06939409923243636|  -0.411391642080337| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313|    2|[-1.1282855413707...|[-1.1282855413707...|              73.0|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.10178170576334064|0.14219939028218284| -0.3492867754665158|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313|    1|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.01947813616931213|0.14219939028218284| -0.2871819088527123| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313|    1|[0.71909117704347...|[0.71909117704347...|15.254098360655737|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.01947813616931213|0.06939409923243636|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|    0|[-1.1282855413707...|[-1.1282855413707...|              73.0|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-0.06282543342471639|0.06939409923243636|-0.24060325889236406|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|    1|[-0.2045971821636...|[-0.2045971821636...|              73.0|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| -0.1451290030187449|0.14219939028218284|-0.20955082558545346|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|    8|[0.71909117704347...|[0.71909117704347...| 15.72347266881029|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-0.06282543342471639| 0.2150046813319293|-0.19402460893199816|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313|   16|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.10178170576334064| 0.2150046813319293|-0.14744595897164992|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313|   52|[-0.2045971821636...|[-0.2045971821636...|15.254098360655737|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.3486924145454262|0.36061526343142225|-0.13191974231819462|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313|   69|[-1.1282855413707...|[-1.1282855413707...|15.254098360655737|
| 1.9506756559862992|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.5132995537334832|0.28780997238167577|-0.08534109235784636|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313|   54|[1.95067565598629...|[1.95067565598629...|15.254098360655737|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.6779066929215403|0.36061526343142225|-0.10086730901130166|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313|   21|[0.41119505730777...|[0.41119505730777...|15.254098360655737|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.1717281104857113|0.28780997238167577|-0.16297217562510521|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313|   15|[0.41119505730777...|[0.41119505730777...|15.254098360655737|
| 1.3348834165148888|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|   1.089424540891683| 0.2150046813319293| -0.1784983922785605|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313|   15|[1.33488341651488...|[1.33488341651488...| 18.88695652173913|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.3363352496737684| 0.2150046813319293|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|   17|[-0.2045971821636...|[-0.2045971821636...| 18.88695652173913|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|    1.25403168007974| 0.2150046813319293| -0.3337605588130782|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313|   12|[0.41119505730777...|[0.41119505730777...| 18.88695652173913|
|-0.2045971821636372|-0.3045476160907193|  0|  1|  0|  0|  0|  0|0.4638614904960641|  1.3363352496737684|0.14219939028218284| -0.3182343421596229| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313|   27|[-0.2045971821636...|[-0.2045971821636...|36.416666666666664|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.3363352496737684|0.06939409923243636|-0.39586542542688175| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313|   47|[-0.2045971821636...|[-0.2045971821636...| 18.88695652173913|
| 1.0269872967791835|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|    1.25403168007974|0.06939409923243636|-0.44244407538722996|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313|  165|[1.02698729677918...|[1.02698729677918...| 18.88695652173913|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.1717281104857113|0.28780997238167577|-0.47349650869414056|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313|  128|[0.71909117704347...|[0.71909117704347...| 18.88695652173913|
| 1.0269872967791835|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.1717281104857113|0.06939409923243636| -0.4579702920406853|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313|   36|[1.02698729677918...|[1.02698729677918...| 18.88695652173913|
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+
only showing top 20 rows

df_reg = july_prediction1_df.select("*").toPandas()
df_reg.to_csv('df_regression.csv')
df_reg

	SPD	GUS	CLR	SCT	BKN	OVC	OBS	POB	VSB	TEMP	DEWP	SLP	ALT	STP	PCP01	count	va_features	features	prediction
0	-1.128286	-0.304548	1	0	0	0	0	0	0.463861	0.184085	0.069394	-0.411392	-0.400542	-0.402984	-0.178119	2	[-1.1282855413707529, -0.3045476160907193, 1.0...	[-1.1282855413707529, -0.3045476160907193, 1.0...	73.000000
1	-0.204597	-0.304548	1	0	0	0	0	0	0.463861	0.101782	0.142199	-0.349287	-0.347805	-0.340400	-0.178119	1	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	15.254098
2	0.719091	-0.304548	1	0	0	0	0	0	0.463861	0.019478	0.142199	-0.287182	-0.295068	-0.293462	-0.178119	1	[0.7190911770434785, -0.3045476160907193, 1.0,...	[0.7190911770434785, -0.3045476160907193, 1.0,...	15.254098
3	-1.128286	-0.304548	1	0	0	0	0	0	0.463861	0.019478	0.069394	-0.256129	-0.242331	-0.246525	-0.178119	0	[-1.1282855413707529, -0.3045476160907193, 1.0...	[-1.1282855413707529, -0.3045476160907193, 1.0...	73.000000
4	-0.204597	-0.304548	1	0	0	0	0	0	0.463861	-0.062825	0.069394	-0.240603	-0.242331	-0.246525	-0.178119	1	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	73.000000
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
727	-0.204597	-0.304548	1	0	0	0	0	0	0.097945	0.924817	1.161473	0.302814	0.337777	0.332374	-0.178119	61	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	22.591036
728	-1.128286	-0.304548	1	0	0	0	0	0	-0.349287	1.089425	1.161473	0.333867	0.337777	0.332374	-0.178119	103	[-1.1282855413707529, -0.3045476160907193, 1.0...	[-1.1282855413707529, -0.3045476160907193, 1.0...	22.591036
729	-1.128286	-0.304548	1	0	0	0	0	0	-0.349287	1.171728	1.161473	0.333867	0.337777	0.332374	-0.178119	64	[-1.1282855413707529, -0.3045476160907193, 1.0...	[-1.1282855413707529, -0.3045476160907193, 1.0...	22.591036
730	-0.204597	-0.304548	1	0	0	0	0	0	-0.349287	1.418639	1.161473	0.380445	0.390514	0.394957	-0.178119	20	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	22.591036
731	-0.204597	-0.304548	1	0	0	0	0	0	0.463861	1.418639	1.088668	0.364919	0.390514	0.394957	-0.178119	12	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	22.591036

732 rows × 19 columns

#2.build the pipline of gradient boost tree regression
from pyspark.ml.regression import GBTRegressor
gbt = GBTRegressor(labelCol="count", featuresCol="features")
gbt_pipeline = Pipeline(stages=[vector, vIndexer, gbt])
gbt_model = gbt_pipeline.fit(train_df)
gbt_prediction = gbt_model.transform(test_df)
gbt_prediction.select("count", "va_features","features","prediction").show()
gbt_rmse = evaluator.evaluate(gbt_prediction)
gbt_rmse   

+-----+--------------------+--------------------+------------------+
|count|         va_features|            features|        prediction|
+-----+--------------------+--------------------+------------------+
|    5|[-0.2045971821636...|[-0.2045971821636...|14.167528541903247|
|    8|[0.41119505730777...|[0.41119505730777...|15.762970386846868|
|    2|[0.41119505730777...|[0.41119505730777...|16.165373708927575|
|    1|[-0.2045971821636...|[-0.2045971821636...|16.165373708927575|
|    0|[-1.1282855413707...|[-1.1282855413707...|14.318582260299902|
|    1|[0.41119505730777...|[0.41119505730777...|12.958096034728644|
|    5|[-1.1282855413707...|[-1.1282855413707...|12.958096034728644|
|    1|[-1.1282855413707...|[-1.1282855413707...| 83.60026986973979|
|   10|[0.71909117704347...|[0.71909117704347...| 67.07887357859991|
|   11|[-1.1282855413707...|[-1.1282855413707...|19.105158819305256|
|   13|[0.71909117704347...|[0.71909117704347...| 66.01947449524297|
|   20|[-1.1282855413707...|[-1.1282855413707...|12.493261239382463|
|   22|[0.71909117704347...|[0.71909117704347...|18.392433998218916|
|   20|[-1.1282855413707...|[-1.1282855413707...| 16.67009126337221|
|   28|[0.41119505730777...|[0.41119505730777...|14.158786115650623|
|   19|[-0.2045971821636...|[-0.2045971821636...|14.158786115650623|
|    9|[-0.2045971821636...|[-0.2045971821636...|15.287032905821656|
|   21|[-0.2045971821636...|[-0.2045971821636...| 29.57280198421807|
|   17|[-0.2045971821636...|[-0.2045971821636...| 16.48369429868547|
|   22|[0.71909117704347...|[0.71909117704347...| 19.82571943908685|
+-----+--------------------+--------------------+------------------+
only showing top 20 rows

38.27806026631097

#realtime streaming of GBTR
realtime_list=realtime_df.collect()
newtrain_list=train_df.collect()

july_prediction2_list=[]
for i in range(int(realtime_df.count()/12)):
    print(i+1) 
    realtimepart_list=realtime_list[i*12:(i*12+12)] 
    realtimepart_df = spark.createDataFrame(realtimepart_list)
    gbt_prediction = gbt_model.transform(realtimepart_df)
    gbt_prediction.select("count", "prediction").show()
    rmse = evaluator.evaluate(gbt_prediction)
    print(rmse)
    july_prediction2_list=july_prediction2_list+gbt_prediction.collect()
    
    newtrain_list=newtrain_list + realtimepart_list
    newtrain_df = spark.createDataFrame(newtrain_list)
    gbt_model = gbt_pipeline.fit(newtrain_df)
    
print(type(july_prediction2_list)) 
print(len(july_prediction2_list))
print(july_prediction2_list[0:20])
july_prediction2_df = spark.createDataFrame(july_prediction2_list)
july_prediction2_df.show()

1
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2| 61.59404460991423|
|    1| 7.778935354101069|
|    1|15.966927358090784|
|    0| 82.31708145370767|
|    1| 82.31708145370767|
|    8|18.014751874827045|
|   16| 11.58732212625986|
|   52| 9.357385562682786|
|   69|12.637939746904813|
|   54| 25.24644542515382|
|   21|11.942369854237393|
|   15|12.256402693047209|
+-----+------------------+

44.008672385049046
2
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   15|23.153034885322995|
|   17|17.922527217444244|
|   12|17.922527217444244|
|   27| 35.94808745540508|
|   47|15.677677873198391|
|  165|15.677677873198391|
|  128|18.427032192739624|
|   36|15.677677873198391|
|   18| 18.72544654958129|
|    7|18.427032192739624|
|   11|18.427032192739624|
|    7|18.427032192739624|
+-----+------------------+

54.921750883178575
3
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2|19.846527437718258|
|    3|13.909832811299347|
|    1|16.549829045586407|
|    2| 30.54390426115427|
|    0|31.900978143890647|
|    5|17.946698864611374|
|   23|17.946698864611374|
|   52|21.243397256743226|
|  102|28.304206011972752|
|   66|21.243397256743226|
|   17|21.243397256743226|
|   14|21.971403113888442|
+-----+------------------+

30.504974674357197
4
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   17|27.873722212475727|
|   15| 27.15229547121602|
|   19|25.828922362070486|
|   25|22.075890808794966|
|   44| 22.87934191261387|
|  112|23.986461882434195|
|  122|  27.4425946824497|
|   28| 35.23801716355055|
|   14|26.529442511048146|
|   13|23.986461882434195|
|    4|22.687616482472023|
|    5| 22.43974587811664|
+-----+------------------+

39.19834033311445
5
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1|22.493375535453147|
|    1| 8.467205610276096|
|    0|15.179736997706383|
|    1| 25.77169878479795|
|    0|31.392109480593565|
|    5|27.166885472527525|
|   17|13.080056670186371|
|   33|13.080056670186371|
|   58| 21.69775339278299|
|   39| 15.52173753852034|
|   17|23.374977609905645|
|   13| 5.938352858406112|
+-----+------------------+

20.837427047512
6
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   17| 17.31520433765312|
|   37|23.344046364450055|
|   44| 20.31139539653864|
|   78|23.344046364450055|
|   67|23.344046364450055|
|   93| 33.34404636445006|
|   36|25.028298531197883|
|   20|24.419405774524314|
|   15|14.185499186036171|
|    7| 27.98190899588396|
|    7|17.409471855873708|
|    6|14.965670587162931|
+-----+------------------+

28.82307585648045
7
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3| 9.345680160666273|
|    0|  16.1849763144682|
|    0|  16.1849763144682|
|    0| 39.12344561416115|
|    0| 33.58753488004096|
|    0|17.000659322451188|
|    2|18.068532493043453|
|    3| 34.65540805063323|
|    2| 33.34656163841736|
|    5| 45.80968249846542|
|    8|29.321078234573953|
|   12|26.930609226234417|
+-----+------------------+

25.977192982822434
8
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   14|11.663619098497314|
|    9| 8.918717408844822|
|   18|19.159338375312238|
|   19|18.535830255961915|
|   19|   8.2952092894945|
|   16|13.350673791213534|
|   14|11.408801906753943|
|   12|11.944988215947735|
|   11| 5.399946336989276|
|    3| 7.503862118884064|
|   15| 4.551951896632436|
|    7| 4.551951896632436|
+-----+------------------+

5.017914844738803
9
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1|  5.89928019933442|
|    1| 6.013534693742454|
|    1| 6.013534693742454|
|    0|14.916755787454331|
|    0| 8.284267298406768|
|    1| 8.284267298406768|
|    6| 12.71872992492693|
|   15|7.2173319049034035|
|   26|15.695174521184395|
|   16| 21.12710921136433|
|   10| 20.16838233532268|
|    8| 20.16838233532268|
+-----+------------------+

8.699265793557398
10
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   15| 32.83481257386646|
|   15|21.594045009780494|
|   16|17.227915575114704|
|   23|17.145078023101057|
|   24| 24.89777575062174|
|   49| 14.14030891507898|
|   21| 15.31831975925873|
|   18|  30.9996693058868|
|   15| 15.31831975925873|
|    7| 15.31831975925873|
|    7| 15.31831975925873|
|   11| 8.717104637712213|
+-----+------------------+

12.774742787185414
11
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    7|14.807302290051968|
|    2|14.251998621748678|
|    1|14.251998621748678|
|    1|14.251998621748678|
|    0| 22.45145603616718|
|    2| 22.45145603616718|
|    2| 22.45145603616718|
|    4| 22.45145603616718|
|    4|23.572593704883946|
|    3|18.200064606637618|
|    3|22.287016712064126|
|   15| 24.84550307863883|
+-----+------------------+

16.64931410779361
12
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   16| 12.59055617278233|
|   14| 42.37805944397618|
|   17| 50.15338411930086|
|   12| 40.11504040917744|
|   17| 40.11504040917744|
|   12|27.738692357869237|
|   18|48.549698223304496|
|   14|23.602074662957087|
|   13|26.667982781175667|
|   13|26.667982781175667|
|    8|23.072059239921238|
|    8|23.072059239921238|
+-----+------------------+

21.09896880672596
13
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    7|15.166028568140357|
|    3|14.803121778561616|
|    1|12.737491296990251|
|    0|16.294597491520662|
|    0|15.166028568140357|
|    0|15.150987625880575|
|    1|13.402334593455851|
|    3|14.002689476196005|
|    4| 13.84930483335112|
|    5| 9.591648073154264|
|    6|11.770759972328099|
|   10|16.957601559574993|
+-----+------------------+

11.34137032853502
14
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   13| 17.98757579830893|
|   17| 42.08298853584196|
|   19|34.943391768760826|
|   10|12.253477075596907|
|   16| 25.27942562272722|
|   20| 13.51819417607071|
|   16|15.574013754685227|
|   12|15.574013754685227|
|    6|18.148434080378337|
|    5|16.304585555849116|
|    8|16.304585555849116|
|    6| 35.14920484302878|
+-----+------------------+

13.688191690018488
15
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    0|13.646863227216727|
|    0|13.646863227216727|
|    0| 39.37214952154402|
|    0| 48.85032011694959|
|    0|12.434778194177149|
|    3|14.059346857703199|
|   18|11.388319803949999|
|   31|12.434778194177149|
|   51| 11.21150615050385|
|   32| 23.54199072777498|
|   30|16.797588571643935|
|    3|16.797588571643935|
+-----+------------------+

24.1413331658727
16
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    9|22.152214214334066|
|   14|17.774530034431333|
|   18|18.359825973817408|
|   20|16.617122673928947|
|   52|15.084641661316937|
|  165|13.229524505303456|
|  118|14.762005517915464|
|   38|15.958227962705529|
|   19|13.731078417040813|
|   12|13.245391909555444|
|    7|12.534855972250748|
|    3|12.534855972250748|
+-----+------------------+

54.688973543228045
17
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    0|14.035021075591773|
|    1|14.035021075591773|
|    0| 21.23362671797738|
|    0|15.599427921604345|
|    0| 21.00712118861493|
|    7|13.787348398783951|
|   20|13.662107153130492|
|   48|13.541127198108807|
|  106|23.443430654272632|
|   68| 30.56017232750517|
|   13|24.744506938809245|
|   20|21.409741525972343|
+-----+------------------+

30.459401036784325
18
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   14| 24.09021006223882|
|   13|22.438238079994537|
|   16|22.438238079994537|
|   21|22.438238079994537|
|   51|22.438238079994537|
|  163|  23.0366046202864|
|  108|  23.0366046202864|
|   33|22.438238079994537|
|   21|23.896364953130924|
|   16|29.393555023899953|
|    5| 24.76302184836721|
|    4| 24.76302184836721|
+-----+------------------+

49.14241559825254
19
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3|20.256010932200194|
|    2| 22.03718853046706|
|    0|20.256010932200194|
|    1|20.256010932200194|
|    0|20.256010932200194|
|    7|20.256010932200194|
|   22|20.766826443328394|
|   59| 20.28388493834195|
|  105| 19.77306942721375|
|  100|18.997178828836933|
|   38|23.713182849262132|
|   14|19.362488170757803|
+-----+------------------+

38.32405795410877
20
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   22|23.721652551989465|
|   21|  26.6728443662679|
|   20| 24.87133160594182|
|   19|31.560198426666766|
|   41|24.114729763373422|
|  147|23.729370146693174|
|  123|31.174838809986518|
|   33|31.174838809986518|
|   20| 32.32451786393887|
|   16|27.311973630035613|
|    4|27.311973630035613|
|    3|24.114729763373422|
+-----+------------------+

46.00880359477889
21
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3| 16.80867227553702|
|    3|20.302567049622592|
|    0|19.884021077918234|
|    1|19.803612968482543|
|    1| 16.80867227553702|
|    6|19.803612968482543|
|   16|18.525819106227686|
|   56| 23.13666059632795|
|   83| 26.63055537041352|
|   62|24.892265316555367|
|   18|24.892265316555367|
|   10|  19.2171123370851|
+-----+------------------+

24.921205566488197
22
+-----+-------------------+
|count|         prediction|
+-----+-------------------+
|   14|  8.984006285004082|
|   12|  9.412254148030385|
|   18|-1.0441821199065564|
|    4| 52.939832871418965|
|   30|  8.136211523629036|
|   79|  10.30226506919652|
|   76|0.18351010956678815|
|    7| 12.116249745824959|
|    5| 58.235952970408206|
|    3|  18.36056965879067|
|    3| 14.464932060681896|
|    1|  53.97221415730735|
+-----+-------------------+

40.58825655938536
23
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    5|15.415188515273485|
|    1|11.968104030390489|
|    1|20.263657898687903|
|    0|12.869105157386103|
|    0|14.636168205017112|
|    3|13.677895690040518|
|   21| 12.96568069173205|
|   52| 9.696909643729965|
|   75| 47.01941563924919|
|   67|51.802006541554675|
|   20| 50.90174653554755|
|   11|57.674898157888244|
+-----+------------------+

24.315925201533894
24
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   19| 77.85514834247051|
|   16| 37.09861947248614|
|   25|41.950204917728044|
|   31| 48.68857950384889|
|   52|  50.0210473080312|
|  128| 78.62607468403225|
|   67|25.400940589652663|
|   35|21.604687096345646|
|   21|27.602702257275794|
|    8|26.785822037123175|
|   11|26.785822037123175|
|    8|26.785822037123175|
+-----+------------------+

28.663993821894177
25
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3|28.333499672434424|
|    2|15.220584112258322|
|    3|17.894003874392624|
|    2|17.894003874392624|
|    0|18.192058529131256|
|    0|18.192058529131256|
|    3|18.604127761322747|
|    6|18.695998627741574|
|    6|18.695998627741574|
|    6|29.930839985709454|
|    5| 32.15630812190821|
|   16|26.115094176144808|
+-----+------------------+

18.099847182770354
26
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    9| 23.75705474722169|
|   19|29.418592365468907|
|   18|17.426175548416712|
|   23|17.426175548416712|
|   21|14.453211122478104|
|   23|22.226347191879128|
|   25|21.162216885845975|
|   18|21.162216885845975|
|   17|20.283125531798653|
|   14| 22.09221627703422|
|   14|14.285106708700011|
|   13|12.428438969021075|
+-----+------------------+

6.471556669552194
27
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    4|15.665911282961106|
|    3| 19.38598955366075|
|    1|15.665911282961106|
|    0| 19.38598955366075|
|    0| 18.25993603329245|
|    3| 18.25993603329245|
|    4| 19.38598955366075|
|    4|20.238445668853764|
|    5| 18.74255177933975|
|    3| 17.53087415532175|
|   13|30.354347649438946|
|   13| 36.35037171153647|
+-----+------------------+

16.604777582934908
28
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   15| 32.49493422431936|
|   18|24.397728223122858|
|   33| 21.71779143797959|
|   20| 24.07200377121302|
|   26| 32.33212326274208|
|   16|39.354789099996424|
|    8|169.99459813648383|
|    9| 171.0654512666404|
|    7|190.13647584365646|
|    3|167.12777182079617|
|    4|21.847749932088565|
|    2|30.105603521859276|
+-----+------------------+

97.96650650754756
29
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    4|  21.3420441458323|
|    1|  21.3420441458323|
|    0| 13.66016139140512|
|    0| 13.66016139140512|
|    0| 13.66016139140512|
|    7| 15.07925101280391|
|   28| 13.66016139140512|
|   68| 13.66016139140512|
|   92|  21.3420441458323|
|   66|  21.3420441458323|
|   18|28.220531376291955|
|   10| 20.78521741788431|
+-----+------------------+

31.232633219770282
30
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   11| 24.86344297823159|
|   18|24.673937734221216|
|   14|24.673937734221216|
|   22| 44.70559408361369|
|   54| 77.34250916279265|
|  158|44.774449220871276|
|  109|30.278432251798428|
|   43| 30.20957711454084|
|   21|26.961455277888174|
|    8|25.077156797946493|
|    8| 24.07540224781928|
|    4|25.077156797946493|
+-----+------------------+

42.4476393184953
31
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1|26.113278117704787|
|    5|26.113278117704787|
|    1|    19.44878323698|
|    0|23.878584489124492|
|    0|    19.44878323698|
|    3|34.888743389756165|
|   25|18.551076614157033|
|   51|30.306395653548208|
|   92|33.256641511354665|
|   76|38.248297845974754|
|   23|17.891290254682694|
|   20|21.906108662504963|
+-----+------------------+

27.01490020406902
32
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   16|15.097568573314803|
|   10|14.890960821187655|
|   22| 17.71888968152936|
|   19|53.681598968755665|
|   53| 69.04375289092079|
|  161| 73.93074699302801|
|  105| 69.04375289092079|
|   37| 63.16588123240809|
|   21| 63.16588123240809|
|   20| 73.93074699302801|
|    5|40.770237644457424|
|    3| 50.75125767040706|
+-----+------------------+

40.112361443881994
33
+-----+-------------------+
|count|         prediction|
+-----+-------------------+
|    2| -8.461823304082023|
|    0|-17.261627833432065|
|    1| 11.227767333352627|
|    2| 19.276493884953073|
|    1| 11.227767333352627|
|    8| 10.211086683279763|
|   19|  6.540971982646021|
|   63| -12.11779087753913|
|   83|-5.0232441112375055|
|   62|   52.9543906063527|
|   25| 35.844101421175424|
|    8|  33.32237331177195|
+-----+-------------------+

35.714884503564335
34
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   13| 21.63952838338922|
|   15|18.008990739783652|
|   15|  41.4432317464839|
|   16|46.901256324661894|
|   39|29.484767262166674|
|  145| 33.98960204807983|
|  110| 32.25371806179067|
|   41| 35.71014773004641|
|    7|0.2950236508234202|
|    5| 63.47522164468109|
|    3|10.612833057045894|
|    3|1.5481397951419447|
+-----+------------------+

44.48643426446519
35
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2|16.705383161247706|
|    1|16.492736704917846|
|    0|3.1132566514570508|
|    0| 4.768375493117132|
|    0| 24.57404727368431|
|    3|13.455914942171239|
|   14|13.987263707766147|
|   47|15.160688517285143|
|   43|21.931200654280172|
|   28|3.3259031077869095|
|    7| 5.243816957269909|
|    9|22.345688813763203|
+-----+------------------+

16.954723488976427
36
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    9|19.163821855191266|
|   12| 12.17315685742773|
|   30|15.631143156118059|
|   19|15.848931349253883|
|   34|11.915344454375198|
|  155|14.218667517313827|
|   93|10.532613429662584|
|   37| 10.78101017201807|
|   15|10.532613429662584|
|    6|10.532613429662584|
|    5|12.129322608228156|
|    5|14.144656037159493|
+-----+------------------+

48.554180649314816
37
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2| 9.924667793193157|
|    5|  8.61683043272282|
|    0|  8.61683043272282|
|    0| 9.785651116368067|
|    0| 10.24354873923144|
|    6|11.464738840771801|
|   14|11.464738840771801|
|   40|12.525002906811029|
|   58|12.480039889729175|
|   59|17.195749324993272|
|   15|15.132411941430528|
|   17|15.132411941430528|
+-----+------------------+

20.34116268656242
38
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    7|17.725242051759945|
|   24|27.593671303686424|
|   31| 66.65078994916074|
|   43| 79.03075683081322|
|   72| 79.03075683081322|
|   91| 68.95186495387749|
|   51|32.397168000083234|
|   30| 35.90662624267324|
|   17|32.397168000083234|
|    8| 35.90662624267324|
|    2| 79.03075683081322|
|    8| 79.03075683081322|
+-----+------------------+

36.06545372592152
39
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    6|24.006820441686795|
|    1| 46.81428320563627|
|    2| 39.57111714738873|
|    0| 36.89695463939994|
|    1| 36.89695463939994|
|    1|44.105219191588915|
|    1| 62.08734677190399|
|    2| 62.08734677190399|
|    4| 46.81428320563627|
|   10|24.006820441686795|
|   12|32.030884118434244|
|   11| 26.10789521476662|
+-----+------------------+

39.09785945416394
40
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   11| 70.02455924576694|
|   12| 50.78493579220319|
|   10| 50.78493579220319|
|   15| 65.73287025717853|
|   10|53.674871023732464|
|   17| 67.94645452274278|
|   10|53.674871023732464|
|   13| 67.94645452274278|
|   11| 65.73287025717853|
|    9|58.617168558511466|
|    4| 65.73287025717853|
|    6| 64.75843838397275|
+-----+------------------+

51.133892556267526
41
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1|4.7165423842459155|
|    2|13.338253040784453|
|    0| 13.07261749930427|
|    0|13.338253040784453|
|    0| 9.915710620518581|
|    1|22.956112919623997|
|    1|13.338253040784453|
|    4|12.431777014324659|
|    5|54.574304797783086|
|    3|17.064025643810538|
|    5| 8.222553249575235|
|    7| 81.93814625596065|
+-----+------------------+

28.24949805211898
42
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   16|10.643192983214792|
|   16|15.399957007953724|
|   16| 27.25754376214232|
|   15| 27.25754376214232|
|   11|61.047643600726666|
|   15| 24.19628826999038|
|   16|16.131369495003916|
|    6|10.362970537655755|
|    7| 8.715798405189002|
|    7| 43.08008904364292|
|    3| 41.90482978082634|
|    2|19.061571198739024|
+-----+------------------+

22.4050329223903
43
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3| 13.49831775154648|
|    2|18.221049244558852|
|    3|14.498983587659742|
|    0|14.498983587659742|
|    0|19.922902901323894|
|    4|19.922902901323894|
|   23|14.364532258791566|
|   53|16.056917737031338|
|   70|18.774212318865313|
|   55|27.421806032579354|
|   16|26.228658810257436|
|    9| 8.559737679767618|
+-----+------------------+

22.903902847685266
44
+-----+-------------------+
|count|         prediction|
+-----+-------------------+
|    9| 12.623980165297931|
|   14| 15.965393494904168|
|   12| 28.466751164498632|
|   14|  6.042019957420359|
|   27|-13.535995643280058|
|   89|   102.363807158359|
|   15|  10.25581261259141|
|    4|  83.90025518374071|
|    9|  9.804248438109669|
|   13|  45.14832201874119|
|    8| 44.646916590394646|
|    6| 12.373465930759883|
+-----+-------------------+

30.27316565978336
45
+-----+-------------------+
|count|         prediction|
+-----+-------------------+
|    0| 11.853576085841517|
|    1|  47.79734581465977|
|    0|-0.9639296011587255|
|    1|-0.7685593458159544|
|    0| 4.6913320612818366|
|    1|   79.1062642148555|
|    0|  4.416717320172759|
|   10|  4.221347064829987|
|   77| 14.023974346652029|
|   63|  16.02276338585752|
|   15| 17.328239393326385|
|   17| 18.518772708848946|
+-----+-------------------+

34.98883282289614
46
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   16|22.489559354147428|
|   13| 18.16934520066622|
|   18|20.008272228001797|
|   22|18.313057708328213|
|   31|21.325353845987767|
|  116|16.744646464223727|
|  138| 18.16934520066622|
|   41|16.995976090342243|
|   20|17.918015574547706|
|    8| 18.16934520066622|
|   11| 18.16934520066622|
|    6| 18.16934520066622|
+-----+------------------+

45.89345351155377
47
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1| 22.03758309964643|
|    0| 5.257660319723647|
|    1| 22.03758309964643|
|    1| 22.03758309964643|
|    0| 22.03758309964643|
|    6|24.524001235768623|
|   26|10.041855610690485|
|   62| 16.65522939338268|
|   90| 22.13120578562024|
|   93| 28.57271707507125|
|   31| 22.76805335617768|
|   22| 16.29294907203198|
+-----+------------------+

33.35712114561442
48
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   28| 17.77364330595932|
|   26|20.631205072684615|
|   36|  28.4729103277732|
|   36|22.254887948428365|
|   54|20.782651061048615|
|  118| 18.67912552905151|
|  110|18.696439316146915|
|   57|22.276748358351814|
|   22|22.276748358351814|
|   16|18.090542901924028|
|    6| 19.62072178417933|
|    6|17.668091257215455|
+-----+------------------+

42.04677520686724
49
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3|  13.7439271454672|
|    6|15.371955018534408|
|    2|13.364591450545284|
|    2|17.672247133270492|
|    2|16.428882447891354|
|    6|16.428882447891354|
|   21| 23.02748796086793|
|   54|18.204943581617478|
|   89|30.977154422328688|
|   71| 31.93754917168908|
|   24|20.269843467063918|
|   13| 20.65326608931397|
+-----+------------------+

24.401847629658526
50
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   13|21.209899583900807|
|   23|16.026536317156275|
|   13| 33.78957278667738|
|   30| 26.19608242690129|
|   55|22.528475246165772|
|  142|16.026536317156275|
|  112| 26.96804019988156|
|   50| 34.65547372972327|
|   27| 32.17546859095647|
|   21| 32.48922068270568|
|   22| 32.48922068270568|
|   13| 17.50042547059527|
+-----+------------------+

45.863361525884436
51
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    8| 33.00715315211512|
|    0|24.552081921265028|
|    1|24.992432103568152|
|    1|18.177685496342068|
|    2|18.654271874937205|
|    7|16.943423504956854|
|   23|16.943423504956854|
|   37|21.963984915644314|
|   71|27.906563456663257|
|   67| 39.16726998886576|
|   18| 29.67145339360266|
|   12| 34.47462384884114|
+-----+------------------+

22.380998447885755
52
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   18|28.823362374879675|
|   22| 25.75748181137665|
|   37|28.009910082756683|
|   42| 9.800595783655854|
|   80| 9.800595783655854|
|  105|28.009910082756683|
|   65|35.622026591645415|
|   29|31.629330597781404|
|   15| 30.30576025872656|
|   23|26.788679265152496|
|   11|26.788679265152496|
|    7|24.638954712532794|
+-----+------------------+

33.89116366399727
53
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    7| 23.27250659656469|
|    2| 35.53697766932646|
|    2|19.480568343214966|
|    1|20.933700352333084|
|    0| 29.46091579708931|
|    0|   36.519386920981|
|    5|30.850187124026455|
|    4| 42.74328249639363|
|    8|39.174966244435225|
|   12|31.865028703856602|
|   10|52.204770044911946|
|   21| 36.45431968837311|
+-----+------------------+

28.65005499182864
54
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   18|43.072072837039876|
|   13|27.746038740736854|
|   13| 21.53527962672211|
|   23|  35.7354749643796|
|   25|37.098053700416074|
|   21|23.393139636097498|
|   25|24.561923351789485|
|   15|29.819378775533323|
|   11|26.308142978904044|
|   11|23.393139636097498|
|   11|23.393139636097498|
|    3|23.393139636097498|
+-----+------------------+

14.172723040585831
55
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2|20.427965414986986|
|    1| 21.17788034214797|
|    2|24.160659650879612|
|    0|18.781294260756358|
|    0|18.940117307768315|
|    1|27.890988292950613|
|    1|27.110470518258325|
|    0|29.202537398055394|
|    4|38.366391575339065|
|    6|28.610088869733026|
|   10|48.365316806672354|
|   10|  42.8027645675872|
+-----+------------------+

26.531664504327317
56
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   13|28.689478418638227|
|   14| 25.66187233410415|
|   21| 25.66187233410415|
|    5|23.886766917960756|
|    0|23.471558349610813|
|    0|16.729877910620456|
|    0|27.923932003496986|
|    0| 25.66187233410415|
|    0| 25.66187233410415|
|    0|22.989597994437695|
|    0|23.271878307481252|
|    0| 8.446404554029758|
+-----+------------------+

20.06169220935807
57
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    0|17.731119540354285|
|    0|11.934976630249048|
|    2| 16.04564928209186|
|    1|11.016758580820744|
|    0|11.016758580820744|
|    6|11.016758580820744|
|   14|13.092901022212162|
|   44|13.999824899477238|
|   92| 8.698079070551664|
|   55| 11.43666438171431|
|   23|10.908948245735894|
|   10|28.355208614106303|
+-----+------------------+

30.443805624685886
58
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   19|16.810435055556397|
|   10| 14.44218606847635|
|   16|17.014796085706312|
|   19|18.987517289177507|
|   47|18.226090347164114|
|  148|17.437798928024865|
|  135|22.777680335151565|
|   41| 31.44808336362216|
|   25| 21.87749389118125|
|    5|23.583764166381034|
|    6|47.487743975063815|
|    3|23.400589491033493|
+-----+------------------+

52.50205935103569
59
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1| 36.36564244287403|
|    1|  22.0335108644563|
|    1| 42.10819464004774|
|    0|21.892830941642284|
|    0|21.549871467702285|
|    8| 17.68859546953034|
|   24|14.294141915634514|
|   46| 20.79121125736992|
|   94|15.120464739690412|
|   47| 22.11415552670455|
|   19|37.679382371896565|
|   19| 48.71945754850123|
+-----+------------------+

33.19603557173788
60
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   11|34.118223812026635|
|   11| 31.87967667791037|
|   25|26.541254575701057|
|   11|19.263069344747134|
|   42| 23.83912041476887|
|  158| 25.80562985125877|
|  113|  28.2282689605956|
|   34|  28.2282689605956|
|   23|  28.2282689605956|
|    8|30.997442093313456|
|   10|  25.0680339530201|
|    5|25.844183298025374|
+-----+------------------+

47.68257720220505
61
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    0| 24.55782268267621|
|    1|18.679868121218988|
|    1|21.483616704089965|
|    3|22.602403304856715|
|    0|21.483616704089965|
|    6|22.602403304856715|
|   24|22.602403304856715|
|   61|17.330501809810468|
|  103|29.671106620703974|
|   64|29.671106620703974|
|   20| 33.36515303635377|
|   12| 23.41681572041636|
+-----+------------------+

30.59170858930127
<class 'list'>
732
[Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.18408527535736915, DEWP=0.06939409923243636, SLP=-0.411391642080337, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=2, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1841, 0.0694, -0.4114, -0.4005, -0.403, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1841, 0.0694, -0.4114, -0.4005, -0.403, -0.1781]), prediction=61.59404460991423), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.14219939028218284, SLP=-0.3492867754665158, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.1422, -0.3493, -0.3478, -0.3404, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.1422, -0.3493, -0.3478, -0.3404, -0.1781]), prediction=7.778935354101069), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.14219939028218284, SLP=-0.2871819088527123, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.1422, -0.2872, -0.2951, -0.2935, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.1422, -0.2872, -0.2951, -0.2935, -0.1781]), prediction=15.966927358090784), Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.06939409923243636, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=0, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.0694, -0.2561, -0.2423, -0.2465, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.0694, -0.2561, -0.2423, -0.2465, -0.1781]), prediction=82.31708145370767), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.06939409923243636, SLP=-0.24060325889236406, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.0694, -0.2406, -0.2423, -0.2465, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.0694, -0.2406, -0.2423, -0.2465, -0.1781]), prediction=82.31708145370767), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.1451290030187449, DEWP=0.14219939028218284, SLP=-0.20955082558545346, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=8, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.1451, 0.1422, -0.2096, -0.2423, -0.2465, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.1451, 0.1422, -0.2096, -0.2423, -0.2465, -0.1781]), prediction=18.014751874827045), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.2150046813319293, SLP=-0.19402460893199816, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=16, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.215, -0.194, -0.1896, -0.1839, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.215, -0.194, -0.1896, -0.1839, -0.1781]), prediction=11.58732212625986), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.2150046813319293, SLP=-0.14744595897164992, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=52, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.215, -0.1474, -0.1369, -0.137, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.215, -0.1474, -0.1369, -0.137, -0.1781]), prediction=9.357385562682786), Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.3486924145454262, DEWP=0.36061526343142225, SLP=-0.13191974231819462, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=69, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.3487, 0.3606, -0.1319, -0.1369, -0.137, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.3487, 0.3606, -0.1319, -0.1369, -0.137, -0.1781]), prediction=12.637939746904813), Row(SPD=1.9506756559862992, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.28780997238167577, SLP=-0.08534109235784636, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=54, va_features=DenseVector([1.9507, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.5133, 0.2878, -0.0853, -0.0841, -0.0901, -0.1781]), features=DenseVector([1.9507, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.5133, 0.2878, -0.0853, -0.0841, -0.0901, -0.1781]), prediction=25.24644542515382), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.6779066929215403, DEWP=0.36061526343142225, SLP=-0.10086730901130166, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=21, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.6779, 0.3606, -0.1009, -0.0841, -0.0901, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.6779, 0.3606, -0.1009, -0.0841, -0.0901, -0.1781]), prediction=11.942369854237393), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.16297217562510521, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=15, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.163, -0.1369, -0.137, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.163, -0.1369, -0.137, -0.1781]), prediction=12.256402693047209), Row(SPD=1.3348834165148888, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.089424540891683, DEWP=0.2150046813319293, SLP=-0.1784983922785605, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=15, va_features=DenseVector([1.3349, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.0894, 0.215, -0.1785, -0.1896, -0.1839, -0.1781]), features=DenseVector([1.3349, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.0894, 0.215, -0.1785, -0.1896, -0.1839, -0.1781]), prediction=23.153034885322995), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.2150046813319293, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=17, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.215, -0.2561, -0.2423, -0.2465, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.215, -0.2561, -0.2423, -0.2465, -0.1781]), prediction=17.922527217444244), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.2150046813319293, SLP=-0.3337605588130782, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=12, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.215, -0.3338, -0.3478, -0.3404, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.215, -0.3338, -0.3478, -0.3404, -0.1781]), prediction=17.922527217444244), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.14219939028218284, SLP=-0.3182343421596229, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=27, va_features=DenseVector([-0.2046, -0.3045, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.1422, -0.3182, -0.2951, -0.2935, -0.1781]), features=DenseVector([-0.2046, -0.3045, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.1422, -0.3182, -0.2951, -0.2935, -0.1781]), prediction=35.94808745540508), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.06939409923243636, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=47, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.0694, -0.3959, -0.4005, -0.403, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.0694, -0.3959, -0.4005, -0.403, -0.1781]), prediction=15.677677873198391), Row(SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.06939409923243636, SLP=-0.44244407538722996, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=165, va_features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.0694, -0.4424, -0.4533, -0.4499, -0.1781]), features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.0694, -0.4424, -0.4533, -0.4499, -0.1781]), prediction=15.677677873198391), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.47349650869414056, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=128, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.4735, -0.4533, -0.4499, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.4735, -0.4533, -0.4499, -0.1781]), prediction=18.427032192739624), Row(SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.06939409923243636, SLP=-0.4579702920406853, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=36, va_features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.0694, -0.458, -0.4533, -0.4499, -0.1781]), features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.0694, -0.458, -0.4533, -0.4499, -0.1781]), prediction=15.677677873198391)]
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+
|                SPD|                GUS|CLR|SCT|BKN|OVC|OBS|POB|               VSB|                TEMP|               DEWP|                 SLP|                 ALT|                 STP|               PCP01|count|         va_features|            features|        prediction|
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.18408527535736915|0.06939409923243636|  -0.411391642080337| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313|    2|[-1.1282855413707...|[-1.1282855413707...| 61.59404460991423|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.10178170576334064|0.14219939028218284| -0.3492867754665158|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313|    1|[-0.2045971821636...|[-0.2045971821636...| 7.778935354101069|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.01947813616931213|0.14219939028218284| -0.2871819088527123| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313|    1|[0.71909117704347...|[0.71909117704347...|15.966927358090784|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.01947813616931213|0.06939409923243636|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|    0|[-1.1282855413707...|[-1.1282855413707...| 82.31708145370767|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-0.06282543342471639|0.06939409923243636|-0.24060325889236406|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|    1|[-0.2045971821636...|[-0.2045971821636...| 82.31708145370767|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| -0.1451290030187449|0.14219939028218284|-0.20955082558545346|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|    8|[0.71909117704347...|[0.71909117704347...|18.014751874827045|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-0.06282543342471639| 0.2150046813319293|-0.19402460893199816|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313|   16|[-0.2045971821636...|[-0.2045971821636...| 11.58732212625986|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.10178170576334064| 0.2150046813319293|-0.14744595897164992|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313|   52|[-0.2045971821636...|[-0.2045971821636...| 9.357385562682786|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.3486924145454262|0.36061526343142225|-0.13191974231819462|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313|   69|[-1.1282855413707...|[-1.1282855413707...|12.637939746904813|
| 1.9506756559862992|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.5132995537334832|0.28780997238167577|-0.08534109235784636|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313|   54|[1.95067565598629...|[1.95067565598629...| 25.24644542515382|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.6779066929215403|0.36061526343142225|-0.10086730901130166|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313|   21|[0.41119505730777...|[0.41119505730777...|11.942369854237393|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.1717281104857113|0.28780997238167577|-0.16297217562510521|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313|   15|[0.41119505730777...|[0.41119505730777...|12.256402693047209|
| 1.3348834165148888|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|   1.089424540891683| 0.2150046813319293| -0.1784983922785605|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313|   15|[1.33488341651488...|[1.33488341651488...|23.153034885322995|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.3363352496737684| 0.2150046813319293|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|   17|[-0.2045971821636...|[-0.2045971821636...|17.922527217444244|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|    1.25403168007974| 0.2150046813319293| -0.3337605588130782|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313|   12|[0.41119505730777...|[0.41119505730777...|17.922527217444244|
|-0.2045971821636372|-0.3045476160907193|  0|  1|  0|  0|  0|  0|0.4638614904960641|  1.3363352496737684|0.14219939028218284| -0.3182343421596229| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313|   27|[-0.2045971821636...|[-0.2045971821636...| 35.94808745540508|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.3363352496737684|0.06939409923243636|-0.39586542542688175| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313|   47|[-0.2045971821636...|[-0.2045971821636...|15.677677873198391|
| 1.0269872967791835|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|    1.25403168007974|0.06939409923243636|-0.44244407538722996|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313|  165|[1.02698729677918...|[1.02698729677918...|15.677677873198391|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.1717281104857113|0.28780997238167577|-0.47349650869414056|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313|  128|[0.71909117704347...|[0.71909117704347...|18.427032192739624|
| 1.0269872967791835|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.1717281104857113|0.06939409923243636| -0.4579702920406853|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313|   36|[1.02698729677918...|[1.02698729677918...|15.677677873198391|
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+
only showing top 20 rows

df_gbtr = july_prediction2_df.select("*").toPandas()
df_gbtr.to_csv('df_gbtr.csv')
df_gbtr

	SPD	GUS	CLR	SCT	BKN	OVC	OBS	POB	VSB	TEMP	DEWP	SLP	ALT	STP	PCP01	count	va_features	features	prediction
0	-1.128286	-0.304548	1	0	0	0	0	0	0.463861	0.184085	0.069394	-0.411392	-0.400542	-0.402984	-0.178119	2	[-1.1282855413707529, -0.3045476160907193, 1.0...	[-1.1282855413707529, -0.3045476160907193, 1.0...	61.594045
1	-0.204597	-0.304548	1	0	0	0	0	0	0.463861	0.101782	0.142199	-0.349287	-0.347805	-0.340400	-0.178119	1	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	7.778935
2	0.719091	-0.304548	1	0	0	0	0	0	0.463861	0.019478	0.142199	-0.287182	-0.295068	-0.293462	-0.178119	1	[0.7190911770434785, -0.3045476160907193, 1.0,...	[0.7190911770434785, -0.3045476160907193, 1.0,...	15.966927
3	-1.128286	-0.304548	1	0	0	0	0	0	0.463861	0.019478	0.069394	-0.256129	-0.242331	-0.246525	-0.178119	0	[-1.1282855413707529, -0.3045476160907193, 1.0...	[-1.1282855413707529, -0.3045476160907193, 1.0...	82.317081
4	-0.204597	-0.304548	1	0	0	0	0	0	0.463861	-0.062825	0.069394	-0.240603	-0.242331	-0.246525	-0.178119	1	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	82.317081
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
727	-0.204597	-0.304548	1	0	0	0	0	0	0.097945	0.924817	1.161473	0.302814	0.337777	0.332374	-0.178119	61	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	17.330502
728	-1.128286	-0.304548	1	0	0	0	0	0	-0.349287	1.089425	1.161473	0.333867	0.337777	0.332374	-0.178119	103	[-1.1282855413707529, -0.3045476160907193, 1.0...	[-1.1282855413707529, -0.3045476160907193, 1.0...	29.671107
729	-1.128286	-0.304548	1	0	0	0	0	0	-0.349287	1.171728	1.161473	0.333867	0.337777	0.332374	-0.178119	64	[-1.1282855413707529, -0.3045476160907193, 1.0...	[-1.1282855413707529, -0.3045476160907193, 1.0...	29.671107
730	-0.204597	-0.304548	1	0	0	0	0	0	-0.349287	1.418639	1.161473	0.380445	0.390514	0.394957	-0.178119	20	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	33.365153
731	-0.204597	-0.304548	1	0	0	0	0	0	0.463861	1.418639	1.088668	0.364919	0.390514	0.394957	-0.178119	12	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	23.416816

732 rows × 19 columns

#3.build the pipline of randomforest regressor
from pyspark.ml.regression import RandomForestRegressor
rf = RandomForestRegressor(numTrees=10, maxDepth=5, seed=101, featuresCol="features",labelCol='count')

rf_pipeline = Pipeline(stages=[vector, vIndexer, rf])
rf_model = rf_pipeline.fit(train_df)
rf_prediction = rf_model.transform(test_df)

rf_prediction.select("count", "va_features","features","prediction").show()
rf_rmse = evaluator.evaluate(rf_prediction)
rf_rmse   

+-----+--------------------+--------------------+------------------+
|count|         va_features|            features|        prediction|
+-----+--------------------+--------------------+------------------+
|    5|[-0.2045971821636...|[-0.2045971821636...| 21.47971511161542|
|    8|[0.41119505730777...|[0.41119505730777...| 17.02752736889692|
|    2|[0.41119505730777...|[0.41119505730777...| 22.58416181643424|
|    1|[-0.2045971821636...|[-0.2045971821636...| 22.58416181643424|
|    0|[-1.1282855413707...|[-1.1282855413707...| 15.90908989152083|
|    1|[0.41119505730777...|[0.41119505730777...| 15.90908989152083|
|    5|[-1.1282855413707...|[-1.1282855413707...| 15.90908989152083|
|    1|[-1.1282855413707...|[-1.1282855413707...|18.188117222794812|
|   10|[0.71909117704347...|[0.71909117704347...|19.605705286862126|
|   11|[-1.1282855413707...|[-1.1282855413707...|18.356472050588092|
|   13|[0.71909117704347...|[0.71909117704347...|20.422502371918718|
|   20|[-1.1282855413707...|[-1.1282855413707...|20.111591250048935|
|   22|[0.71909117704347...|[0.71909117704347...|19.516862400089632|
|   20|[-1.1282855413707...|[-1.1282855413707...|20.593926512015933|
|   28|[0.41119505730777...|[0.41119505730777...|29.915397462244506|
|   19|[-0.2045971821636...|[-0.2045971821636...|29.915397462244506|
|    9|[-0.2045971821636...|[-0.2045971821636...|26.781399677083932|
|   21|[-0.2045971821636...|[-0.2045971821636...| 32.25658484374382|
|   17|[-0.2045971821636...|[-0.2045971821636...|24.120640505794007|
|   22|[0.71909117704347...|[0.71909117704347...| 32.58326276355699|
+-----+--------------------+--------------------+------------------+
only showing top 20 rows

28.453972138637624

#realtime streaming of random forest fregressor
realtime_list=realtime_df.collect()
newtrain_list=train_df.collect()

july_prediction3_list=[]
for i in range(int(realtime_df.count()/12)):
    print(i+1) 
    realtimepart_list=realtime_list[i*12:(i*12+12)] 
    realtimepart_df = spark.createDataFrame(realtimepart_list)
    rf_prediction = rf_model.transform(realtimepart_df)
    rf_prediction.select("count", "prediction").show()
    rmse = evaluator.evaluate(rf_prediction)
    print(rmse)
    july_prediction3_list=july_prediction3_list+rf_prediction.collect()
    
    newtrain_list=newtrain_list + realtimepart_list
    newtrain_df = spark.createDataFrame(newtrain_list)
    rf_model = rf_pipeline.fit(newtrain_df)
    
print(type(july_prediction3_list)) 
print(len(july_prediction3_list))
print(july_prediction3_list[0:20])
july_prediction3_df = spark.createDataFrame(july_prediction3_list)
july_prediction3_df.show()

1
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2| 17.95700665994376|
|    1| 17.28247725096573|
|    1| 18.70006531503304|
|    0|18.188117222794812|
|    1|18.188117222794812|
|    8|19.520817025119957|
|   16| 17.28247725096573|
|   52| 17.28247725096573|
|   69|20.111591250048935|
|   54|22.180979483401217|
|   21|20.111591250048935|
|   15|20.111591250048935|
+-----+------------------+

22.782304886893844
2
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   15|23.780114059765907|
|   17|21.326904911897515|
|   12|22.693524979302715|
|   27|26.835392583369064|
|   47|21.326904911897515|
|  165|22.484917237808524|
|  128|22.484917237808524|
|   36|22.484917237808524|
|   18|21.326904911897515|
|    7|21.326904911897515|
|   11|22.693524979302715|
|    7|21.326904911897515|
+-----+------------------+

52.48354114310918
3
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2|23.161749925041597|
|    3|21.830413519511644|
|    1|23.161749925041597|
|    2|27.924742110051888|
|    0|25.205239412362978|
|    5|21.830413519511644|
|   23|21.830413519511644|
|   52|21.830413519511644|
|  102|23.161749925041597|
|   66| 24.48024740655483|
|   17| 24.48024740655483|
|   14| 29.24323959156512|
+-----+------------------+

31.653657404847646
4
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   17|27.879421552253245|
|   15|26.534984052253247|
|   19|26.534984052253247|
|   25|30.246329494553926|
|   44| 28.90189199455392|
|  112| 28.22831667694956|
|  122|30.246329494553926|
|   28|29.582257723048855|
|   14|29.572754176949566|
|   13| 28.22831667694956|
|    4|21.247351836035325|
|    5|27.879421552253245|
+-----+------------------+

37.9670431519968
5
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1|   26.546641109065|
|    1| 29.39209630848108|
|    0|28.109720829553872|
|    1| 34.76398564242925|
|    0| 37.55731897576258|
|    5| 41.98321858494312|
|   17| 26.99969666462055|
|   33| 26.99969666462055|
|   58|35.402385251609786|
|   39|26.825481234456852|
|   17|   26.546641109065|
|   13|26.825481234456852|
+-----+------------------+

24.57025091903441
6
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   17| 26.57770885707053|
|   37|  26.1666666734477|
|   44| 26.57770885707053|
|   78|  26.1666666734477|
|   67|25.265842497623527|
|   93| 31.06597868493288|
|   36| 26.57770885707053|
|   20| 26.57770885707053|
|   15|22.786670988907836|
|    7| 23.02980010673106|
|    7|23.440842290353892|
|    6|  20.4879840239389|
+-----+------------------+

28.441054138413175
7
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3|  20.2117554165253|
|    0|  20.2117554165253|
|    0|  20.2117554165253|
|    0|28.451229100735826|
|    0|15.592567683412991|
|    0|16.671048696071217|
|    2|16.671048696071217|
|    3|15.592567683412991|
|    2|18.970519186626362|
|    5| 32.11269474430447|
|    8|22.511225907080444|
|   12|22.511225907080444|
+-----+------------------+

18.622946918934982
8
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   14| 24.72920586002534|
|    9| 24.72920586002534|
|   18|27.846175590836975|
|   19| 26.18101943651977|
|   19| 32.43337252669201|
|   16| 24.72920586002534|
|   14| 24.72920586002534|
|   12| 23.91393440348562|
|   11| 19.61869724333341|
|    3|22.019267043903213|
|   15|14.088481737275302|
|    7|14.088481737275302|
+-----+------------------+

11.22438607443106
9
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1|13.567562901886783|
|    1|13.567562901886783|
|    1|13.567562901886783|
|    0|17.294540449899173|
|    0|30.453132035400216|
|    1|30.453132035400216|
|    6| 21.67051246575619|
|   15|30.453132035400216|
|   26| 35.79923715173716|
|   16|35.351737151737154|
|   10| 40.05188144713975|
|    8| 40.05188144713975|
+-----+------------------+

21.317305160587576
10
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   15| 31.28141366480627|
|   15|24.347247813627167|
|   16|19.984475863355218|
|   23| 41.60708329590126|
|   24|26.092107661943505|
|   49|16.601345372596736|
|   21| 17.89896580771717|
|   18| 29.25842162755442|
|   15| 17.89896580771717|
|    7| 17.89896580771717|
|    7| 17.89896580771717|
|   11| 17.89896580771717|
+-----+------------------+

13.535594800889863
11
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    7| 17.07627712439396|
|    2| 17.07627712439396|
|    1| 17.07627712439396|
|    1| 17.07627712439396|
|    0|27.334862383310753|
|    2|27.334862383310753|
|    2|27.334862383310753|
|    4|27.781529049977422|
|    4|29.138009729791435|
|    3|33.929342108792284|
|    3| 42.59558548736719|
|   15|33.957490249271956|
+-----+------------------+

24.07960021385361
12
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   16| 23.02464323700992|
|   14| 32.43283341225508|
|   17| 30.50307150749317|
|   12|23.950169268326782|
|   17|23.950169268326782|
|   12|29.206919780569557|
|   18|29.795027178102238|
|   14|24.797875612587887|
|   13| 26.54940000283179|
|   13| 26.54940000283179|
|    8| 21.81702723102406|
|    8| 21.81702723102406|
+-----+------------------+

13.112802696979612
13
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    7| 21.76879202344602|
|    3|24.838391115943704|
|    1| 21.76879202344602|
|    0|28.358648208908864|
|    0| 21.76879202344602|
|    0|27.619662190420883|
|    1|24.630960917124543|
|    3|23.559576606383537|
|    4|23.559576606383537|
|    5|28.754806861485935|
|    6| 33.65462841177735|
|   10|23.559576606383537|
+-----+------------------+

22.442410040058313
14
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   13|18.245867589154123|
|   17|21.595285095218273|
|   19|20.195224523186898|
|   10| 16.11260245228266|
|   16| 19.91623070147463|
|   20| 17.49765235624964|
|   16|20.410005364904016|
|   12|20.410005364904016|
|    6|20.410005364904016|
|    5|20.410005364904016|
|    8|20.410005364904016|
|    6| 24.21363361409599|
+-----+------------------+

9.70422755388818
15
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    0|17.895940368144554|
|    0|17.895940368144554|
|    0| 21.73330479941315|
|    0|22.474389528686856|
|    0| 16.36773726125839|
|    3|13.427768691567673|
|   18|15.156509517987171|
|   31| 16.36773726125839|
|   51|11.168137887567338|
|   32|   18.140879512793|
|   30|17.895940368144554|
|    3|16.817199541725053|
+-----+------------------+

19.00190679510246
16
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    9|17.933899261835037|
|   14|16.243725325161158|
|   18| 17.22646892127776|
|   20|16.243725325161158|
|   52|16.243725325161158|
|  165|16.243725325161158|
|  118|16.243725325161158|
|   38|16.243725325161158|
|   19|16.243725325161158|
|   12|16.243725325161158|
|    7|16.243725325161158|
|    3|16.243725325161158|
+-----+------------------+

53.71267784812599
17
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    0| 17.33279456900212|
|    1| 17.33279456900212|
|    0| 18.28960897824345|
|    0| 17.33279456900212|
|    0| 16.74367382021152|
|    7| 15.78685941097019|
|   20| 15.78685941097019|
|   48| 17.33279456900212|
|  106| 18.28960897824345|
|   68| 18.28960897824345|
|   13|18.174492897269598|
|   20|18.461567684168447|
+-----+------------------+

32.54648839671255
18
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   14|22.039371597686443|
|   13| 16.42593255059399|
|   16| 16.42593255059399|
|   21| 16.42593255059399|
|   51| 16.42593255059399|
|  163| 16.42593255059399|
|  108| 16.42593255059399|
|   33| 16.42593255059399|
|   21| 16.42593255059399|
|   16| 16.42593255059399|
|    5| 16.42593255059399|
|    4|15.348163324378849|
+-----+------------------+

51.41152692397727
19
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3| 16.16907433066968|
|    2| 16.93563539513982|
|    0|17.471539316664078|
|    1|17.471539316664078|
|    0| 16.16907433066968|
|    7| 16.16907433066968|
|   22| 16.16907433066968|
|   59| 16.16907433066968|
|  105| 16.16907433066968|
|  100| 16.16907433066968|
|   38| 22.17107905728583|
|   14| 16.16907433066968|
+-----+------------------+

39.116151682193
20
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   22|20.788109318197264|
|   21|19.758060191762564|
|   20| 20.39889088929084|
|   19|22.466424863659892|
|   41|18.241034577071964|
|  147|19.042648005382638|
|  123|19.042648005382638|
|   33|19.042648005382638|
|   20|19.758060191762564|
|   16| 19.68347870291091|
|    4| 19.68347870291091|
|    3|18.241034577071964|
+-----+------------------+

48.64845547672396
21
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3|19.914905929234617|
|    3|20.679428706848704|
|    0|19.914905929234617|
|    1| 20.47928391053606|
|    1|19.914905929234617|
|    6| 20.47928391053606|
|   16|19.914905929234617|
|   56|19.914905929234617|
|   83|21.391357130643264|
|   62|23.225932214308436|
|   18|23.225932214308436|
|   10|27.559935620800264|
+-----+------------------+

27.224867567180077
22
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   14|22.921345582952068|
|   12| 21.86261058631818|
|   18|19.891382454916194|
|    4| 7.647371393352563|
|   30| 9.706656484266452|
|   79|11.687201964323616|
|   76|  11.8031621630797|
|    7|10.692507175201847|
|    5|15.608244880119878|
|    3|10.692507175201847|
|    3|12.430420107340131|
|    1| 25.29739773689922|
+-----+------------------+

29.043715892973058
23
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    5|31.952847474221624|
|    1|32.834946237128435|
|    1|31.972051932450874|
|    0|32.834946237128435|
|    0|31.952847474221624|
|    3| 38.91723534283709|
|   21| 33.85692639662724|
|   52|31.952847474221624|
|   75| 39.11804302597094|
|   67|39.544031335680174|
|   20| 39.71305727098518|
|   11|39.544031335680174|
+-----+------------------+

28.723181730642832
24
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   19|  43.4945762166063|
|   16| 33.78786653619724|
|   25| 31.72998314052315|
|   31|39.924237003580515|
|   52| 35.97629389004166|
|  128| 35.68050013156134|
|   67| 31.19478618426004|
|   35| 27.71085766974151|
|   21|24.059289737120075|
|    8| 20.20076681647911|
|   11| 20.20076681647911|
|    8| 20.20076681647911|
+-----+------------------+

31.021299229177675
25
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3|21.993243009875435|
|    2|22.360698841740316|
|    3|20.336824428456847|
|    2|20.663458320880217|
|    0|19.799846509038698|
|    0|19.799846509038698|
|    3|19.799846509038698|
|    6|19.473212616615328|
|    6|19.473212616615328|
|    6| 22.82687171889401|
|    5|21.129631198033913|
|   16|19.894180416930958|
+-----+------------------+

16.86224172863294
26
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    9|21.917931957690445|
|   19|30.688065894151208|
|   18|21.917931957690445|
|   23|21.917931957690445|
|   21|23.009624030861175|
|   23|27.081154312211133|
|   25|25.989462239040403|
|   18|25.989462239040403|
|   17|25.989462239040403|
|   14|25.989462239040403|
|   14|21.917931957690445|
|   13|18.495336699303152|
+-----+------------------+

7.761871940204926
27
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    4|19.965663440096872|
|    3|21.886280219563098|
|    1|21.948991130699206|
|    0|21.948991130699206|
|    0| 20.02837435123298|
|    3| 20.02837435123298|
|    4| 20.02837435123298|
|    4| 20.02837435123298|
|    5|19.965663440096872|
|    3| 21.61331034178032|
|   13| 31.83742080767338|
|   13|26.698174831498466|
+-----+------------------+

17.911928362664405
28
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   15|32.144453558657304|
|   18|23.153661794523863|
|   33| 24.85748763580658|
|   20|26.837487635806582|
|   26|26.837487635806582|
|   16| 46.29597325757062|
|    8|58.035201647694876|
|    9| 57.00551984869173|
|    7|  36.0103457739402|
|    3|27.131774345368775|
|    4|20.567060629571916|
|    2| 28.46014946176762|
+-----+------------------+

26.70626126014692
29
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    4|19.896479210420246|
|    1|19.896479210420246|
|    0|18.206799638084142|
|    0|18.206799638084142|
|    0|18.206799638084142|
|    7|17.418482042089963|
|   28|18.206799638084142|
|   68|18.206799638084142|
|   92|19.896479210420246|
|   66|19.896479210420246|
|   18|19.896479210420246|
|   10| 22.14807879642476|
+-----+------------------+

31.30984505974298
30
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   11|21.868568243631536|
|   18| 25.75312153550439|
|   14| 25.75312153550439|
|   22|31.072224969648158|
|   54|25.568506150889007|
|  158|31.072224969648158|
|  109|21.868568243631536|
|   43|21.868568243631536|
|   21| 25.34135091176194|
|    8| 21.59173060227451|
|    8| 21.59173060227451|
|    4|20.728094238638143|
+-----+------------------+

46.56917638640825
31
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1|19.363358875173116|
|    5|19.363358875173116|
|    1|18.587243120670582|
|    0|19.363358875173116|
|    0|18.587243120670582|
|    3|18.587243120670582|
|   25|18.587243120670582|
|   51|19.363358875173116|
|   92|19.363358875173116|
|   76|21.048926962056992|
|   23|24.983312814560044|
|   20|23.947125651100873|
+-----+------------------+

30.514250051954182
32
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   16| 24.06934109937149|
|   10| 24.06934109937149|
|   22|28.002046067257776|
|   19|31.199787160773447|
|   53| 40.96659600133227|
|  161| 40.96659600133227|
|  105| 40.96659600133227|
|   37| 40.96659600133227|
|   21| 40.96659600133227|
|   20| 32.90659600133227|
|    5|28.002046067257776|
|    3|25.589749446481804|
+-----+------------------+

41.55457021281788
33
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2|24.414352823715976|
|    0|24.414352823715976|
|    1| 17.50700693628664|
|    2| 17.50700693628664|
|    1|18.331513586906052|
|    8|18.937469715491957|
|   19| 17.50700693628664|
|   63|24.500670280722186|
|   83| 21.32568613195813|
|   62| 31.17009256502762|
|   25| 31.77004534882723|
|    8|21.459445075110192|
+-----+------------------+

26.609903798262998
34
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   13| 20.83469994509859|
|   15|22.606623550582178|
|   15|22.606623550582178|
|   16| 28.83103499369779|
|   39|22.606623550582178|
|  145| 31.69497478580727|
|  110|35.267052707885185|
|   41|35.267052707885185|
|    7|  10.8049533153874|
|    5|16.219372612879294|
|    3| 9.659282895548156|
|    3| 7.050669277828713|
+-----+------------------+

40.07024770489666
35
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2|15.633681012676925|
|    1|15.633681012676925|
|    0|11.940032410588929|
|    0|25.757080828042064|
|    0|  22.2595864239091|
|    3| 17.85032289930304|
|   14| 17.85032289930304|
|   47|16.666690036658625|
|   43|16.725393549717648|
|   28|10.351833293043319|
|    7|11.938428147994165|
|    9|  22.2595864239091|
+-----+------------------+

18.391544387213806
36
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    9|18.468463871984543|
|   12|13.411697096588933|
|   30|18.468463871984543|
|   19|17.941721546268724|
|   34|19.536415454852325|
|  155| 18.23725980140152|
|   93| 18.23725980140152|
|   37|19.536415454852325|
|   15| 18.23725980140152|
|    6| 18.23725980140152|
|    5|19.114640753782474|
|    5|18.468463871984543|
+-----+------------------+

46.16481384523791
37
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2|17.281471537356932|
|    5| 18.58554624024318|
|    0| 18.58554624024318|
|    0|18.658553035260166|
|    0|17.354478332373922|
|    6|17.354478332373922|
|   14|17.354478332373922|
|   40|18.658553035260166|
|   58| 18.58554624024318|
|   59|18.585567122607483|
|   15|17.354478332373922|
|   17|17.556296514192102|
+-----+------------------+

20.818951969382805
38
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    7| 31.12172804552434|
|   24|33.607053015942014|
|   31| 36.75324469902248|
|   43|36.876284081612425|
|   72| 56.14429754962589|
|   91| 51.13620766198544|
|   51| 46.21781048052706|
|   30|  37.0011438138604|
|   17| 46.21781048052706|
|    8|  37.0011438138604|
|    2| 41.36829110840206|
|    8| 41.36829110840206|
+-----+------------------+

24.178529614643175
39
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    6| 31.77804720118821|
|    1| 39.79377807997936|
|    2| 39.79377807997936|
|    0| 39.79377807997936|
|    1| 39.79377807997936|
|    1| 33.10203260510118|
|    1| 39.79377807997936|
|    2| 37.89156531402192|
|    4| 37.89156531402192|
|   10| 31.77804720118821|
|   12| 36.73965514144863|
|   11|30.007524813128505|
+-----+------------------+

33.04831922042469
40
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   11|29.278555601664003|
|   12|29.278555601664003|
|   10| 30.79882104321581|
|   15|30.341182434910536|
|   10|31.364932154326926|
|   17| 52.70232366366745|
|   10| 30.79882104321581|
|   13| 52.13621255255633|
|   11|30.341182434910536|
|    9|29.278555601664003|
|    4|28.194118745911368|
|    6| 30.79882104321581|
+-----+------------------+

24.120505292588813
41
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1| 25.41849531546238|
|    2| 26.12180206398596|
|    0|25.854213131179637|
|    0| 26.12180206398596|
|    0| 25.95111590582435|
|    1|28.198174729353763|
|    1|22.852812248184897|
|    4|22.626873338545472|
|    5| 35.81000463441768|
|    3|23.981578098873506|
|    5| 24.69467060950064|
|    7| 35.89621207170452|
+-----+------------------+

24.797469955757144
42
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   16|24.638419516964245|
|   16|  22.4504804184753|
|   16| 19.07708343209191|
|   15|31.096080495952197|
|   11|31.096080495952197|
|   15| 33.33732206688127|
|   16|27.318662236657122|
|    6| 22.00401559677136|
|    7|23.958302927401533|
|    7|25.616084980791037|
|    3|25.616084980791037|
|    2|22.639605523067424|
+-----+------------------+

16.022788394683694
43
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3|20.624866490376515|
|    2|   18.830466616899|
|    3| 19.51312084374124|
|    0| 19.51312084374124|
|    0| 19.51312084374124|
|    4| 19.51312084374124|
|   23| 20.09409255793394|
|   53| 20.09409255793394|
|   70| 24.33942469585708|
|   55|23.474040080472463|
|   16|23.474040080472463|
|    9| 20.53889975440978|
+-----+------------------+

22.78118592364291
44
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    9|23.039572345792557|
|   14|21.123749270158516|
|   12|21.586753624529727|
|   14|17.781152778804582|
|   27|14.816445266419738|
|   89|27.139402481614148|
|   15| 11.39852909995315|
|    4|15.795331878369353|
|    9|15.737296598018935|
|   13|18.827797985253717|
|    8|22.770710725406097|
|    6| 21.23573166828107|
+-----+------------------+

20.43483370856744
45
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    0| 9.172699837440224|
|    1|12.072032937386512|
|    0| 9.818781863930813|
|    1|  8.83965780735877|
|    0| 7.895679890067337|
|    1|10.391681268414075|
|    0| 9.288392557658645|
|   10| 6.948912038178125|
|   77|25.973230919748865|
|   63|22.952128157712565|
|   15|22.952128157712565|
|   17|20.121900453507312|
+-----+------------------+

20.18925387856117
46
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   16|21.914689129581355|
|   13|20.840905247250486|
|   18|20.840905247250486|
|   22| 20.18974549246884|
|   31|28.272048575292057|
|  116|19.613818575523517|
|  138|20.840905247250486|
|   41|18.487686096891036|
|   20| 19.71477276861801|
|    8|18.487686096891036|
|   11|18.487686096891036|
|    6|18.487686096891036|
+-----+------------------+

44.684717015539235
47
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1| 20.60289278065635|
|    0|10.771993070947676|
|    1|15.897065819754715|
|    1|21.216740233513093|
|    0| 20.60289278065635|
|    6| 20.60289278065635|
|   26| 20.60289278065635|
|   62| 20.60289278065635|
|   90| 20.60289278065635|
|   93| 20.60289278065635|
|   31| 20.60289278065635|
|   22| 20.60289278065635|
+-----+------------------+

33.76249483279547
48
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   28|  20.8339090256351|
|   26|  25.7849651291788|
|   36| 21.68352764085801|
|   36| 21.06952174726245|
|   54|21.919140362485358|
|  118|22.428803803648705|
|  110|21.579185188425797|
|   57|21.579185188425797|
|   22|21.579185188425797|
|   16|  20.8339090256351|
|    6|  20.8339090256351|
|    6| 20.74708065434801|
+-----+------------------+

41.00432332819352
49
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    3| 20.63056911190121|
|    6| 20.63056911190121|
|    2|19.920861314996824|
|    2| 21.45490371837408|
|    2|20.483909037997247|
|    6|20.483909037997247|
|   21|19.925827597972223|
|   54|19.920861314996824|
|   89| 20.63056911190121|
|   71| 20.63056911190121|
|   24| 20.63056911190121|
|   13|  19.3627798749718|
+-----+------------------+

29.157357280678443
50
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   13|22.233989579505526|
|   23|21.511608671955162|
|   13|20.308637462688132|
|   30|21.483500957213625|
|   55|22.304147237163185|
|  142|20.690962392005602|
|  112|19.100369263007938|
|   50|19.670246312188265|
|   27|19.670246312188265|
|   21|19.670246312188265|
|   22|19.670246312188265|
|   13|19.670246312188265|
+-----+------------------+

46.23669090038388
51
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    8|21.712050281705583|
|    0|27.368322469030023|
|    1|27.368322469030023|
|    1|27.368322469030023|
|    2|22.702069890340788|
|    7|22.702069890340788|
|   23|22.702069890340788|
|   37|27.368322469030023|
|   71|27.368322469030023|
|   67|25.513998204486775|
|   18| 29.09090042529192|
|   12| 29.09090042529192|
+-----+------------------+

24.384984292062708
52
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   18| 34.04332621280515|
|   22|  29.4820876241371|
|   37|  28.3277797825316|
|   42|26.985921566287676|
|   80|26.985921566287676|
|  105|  28.3277797825316|
|   65|28.090853799565952|
|   29|23.905569178471502|
|   15|24.754936291318824|
|   23|25.369007275815555|
|   11|25.369007275815555|
|    7|22.924799947702272|
+-----+------------------+

30.621613650572503
53
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    7| 28.07452746947276|
|    2|29.654088735572753|
|    2| 28.68109714969384|
|    1|29.092053891174135|
|    0| 28.07452746947276|
|    0| 28.07452746947276|
|    5| 28.07452746947276|
|    4|33.930404134972186|
|    8| 31.84437513098274|
|   12|29.698623571395217|
|   10| 36.00991424403212|
|   21|  33.1304197041641|
+-----+------------------+

24.86632275215677
54
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   18| 36.17682417328529|
|   13|33.965268617729734|
|   13|28.395117061389158|
|   23|28.350771307263983|
|   25|33.965268617729734|
|   21|23.697831672982865|
|   25|22.286863074746844|
|   15|23.873619363553168|
|   11| 25.01590559217603|
|   11|23.697831672982865|
|   11|23.697831672982865|
|    3|23.697831672982865|
+-----+------------------+

13.406006370863338
55
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    2|21.947450904015394|
|    1|24.658762048583725|
|    2|26.048159991375105|
|    0|22.412838130282513|
|    0|24.658762048583725|
|    1|24.658762048583725|
|    1|22.993532982875173|
|    0|22.412838130282513|
|    4|27.790645867081317|
|    6|24.578892160350414|
|   10|27.954859416568667|
|   10|30.418192749902005|
+-----+------------------+

22.065393608823246
56
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   13|25.296200910552567|
|   14|22.853343767695424|
|   21|22.853343767695424|
|    5| 26.96609048688663|
|    0| 26.96609048688663|
|    0| 25.01338125985676|
|    0| 26.96609048688663|
|    0|24.273624892193414|
|    0|24.273624892193414|
|    0|25.428814288497016|
|    0|23.593100135358764|
|    0|13.975409910358763|
+-----+------------------+

21.15915949129792
57
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    0|11.229757938840887|
|    0|19.003721793491255|
|    2| 19.77197598189801|
|    1|19.003721793491255|
|    0|19.003721793491255|
|    6|19.003721793491255|
|   14| 19.77197598189801|
|   44|20.829686286119937|
|   92|20.829686286119937|
|   55|24.892231011774246|
|   23| 26.03385927598315|
|   10| 23.44270159147279|
+-----+------------------+

26.440570525668903
58
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   19|26.192541558645406|
|   10|22.351081424297604|
|   16| 24.58704173159594|
|   19|27.985351655209463|
|   47| 23.55601377417769|
|  148|22.351081424297604|
|  135| 26.65363681507511|
|   41|24.017109030607394|
|   25| 23.55601377417769|
|    5| 23.55601377417769|
|    6|26.163019472157732|
|    3| 23.55601377417769|
+-----+------------------+

49.91566865465443
59
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    1|25.427586867849264|
|    1| 25.32068422551108|
|    1|25.427586867849264|
|    0|23.285857633437704|
|    0|21.094120938759833|
|    8| 20.19868672834622|
|   24|16.735529795859584|
|   46|20.403374597440347|
|   94|20.734997436734247|
|   47|20.488333516165117|
|   19|21.094120938759833|
|   19|30.044300202961068|
+-----+------------------+

28.615062281573945
60
+-----+------------------+
|count|        prediction|
+-----+------------------+
|   11|  24.7375041951986|
|   11|27.726757807004862|
|   25| 24.74476628158113|
|   11|19.371271982491365|
|   42|  23.3966833272657|
|  158|26.068227811018346|
|  113|26.067805152790942|
|   34| 26.97680180130372|
|   23| 26.97680180130372|
|    8|24.737926853426007|
|   10|24.280897326525128|
|    5|24.737926853426007|
+-----+------------------+

47.246479091026174
61
+-----+------------------+
|count|        prediction|
+-----+------------------+
|    0| 21.56314553102278|
|    1|20.818676977580957|
|    1|19.420193669996326|
|    3| 20.36095178976571|
|    0|19.420193669996326|
|    6|21.453041204892788|
|   24| 20.36095178976571|
|   61| 19.63766517009006|
|  103| 20.34608303102278|
|   64| 20.34608303102278|
|   20| 21.12524619070971|
|   12| 22.37893056591924|
+-----+------------------+

32.51256685123153
<class 'list'>
732
[Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.18408527535736915, DEWP=0.06939409923243636, SLP=-0.411391642080337, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=2, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1841, 0.0694, -0.4114, -0.4005, -0.403, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1841, 0.0694, -0.4114, -0.4005, -0.403, -0.1781]), prediction=17.95700665994376), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.14219939028218284, SLP=-0.3492867754665158, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.1422, -0.3493, -0.3478, -0.3404, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.1422, -0.3493, -0.3478, -0.3404, -0.1781]), prediction=17.28247725096573), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.14219939028218284, SLP=-0.2871819088527123, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.1422, -0.2872, -0.2951, -0.2935, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.1422, -0.2872, -0.2951, -0.2935, -0.1781]), prediction=18.70006531503304), Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.01947813616931213, DEWP=0.06939409923243636, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=0, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.0694, -0.2561, -0.2423, -0.2465, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.0195, 0.0694, -0.2561, -0.2423, -0.2465, -0.1781]), prediction=18.188117222794812), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.06939409923243636, SLP=-0.24060325889236406, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=1, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.0694, -0.2406, -0.2423, -0.2465, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.0694, -0.2406, -0.2423, -0.2465, -0.1781]), prediction=18.188117222794812), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.1451290030187449, DEWP=0.14219939028218284, SLP=-0.20955082558545346, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=8, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.1451, 0.1422, -0.2096, -0.2423, -0.2465, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.1451, 0.1422, -0.2096, -0.2423, -0.2465, -0.1781]), prediction=19.520817025119957), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=-0.06282543342471639, DEWP=0.2150046813319293, SLP=-0.19402460893199816, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=16, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.215, -0.194, -0.1896, -0.1839, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, -0.0628, 0.215, -0.194, -0.1896, -0.1839, -0.1781]), prediction=17.28247725096573), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.10178170576334064, DEWP=0.2150046813319293, SLP=-0.14744595897164992, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=52, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.215, -0.1474, -0.1369, -0.137, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.1018, 0.215, -0.1474, -0.1369, -0.137, -0.1781]), prediction=17.28247725096573), Row(SPD=-1.1282855413707529, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.3486924145454262, DEWP=0.36061526343142225, SLP=-0.13191974231819462, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=69, va_features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.3487, 0.3606, -0.1319, -0.1369, -0.137, -0.1781]), features=DenseVector([-1.1283, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.3487, 0.3606, -0.1319, -0.1369, -0.137, -0.1781]), prediction=20.111591250048935), Row(SPD=1.9506756559862992, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.5132995537334832, DEWP=0.28780997238167577, SLP=-0.08534109235784636, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=54, va_features=DenseVector([1.9507, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.5133, 0.2878, -0.0853, -0.0841, -0.0901, -0.1781]), features=DenseVector([1.9507, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.5133, 0.2878, -0.0853, -0.0841, -0.0901, -0.1781]), prediction=22.180979483401217), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=0.6779066929215403, DEWP=0.36061526343142225, SLP=-0.10086730901130166, ALT=-0.08411954340779061, STP=-0.09006563447177515, PCP01=-0.17811912685987313, count=21, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.6779, 0.3606, -0.1009, -0.0841, -0.0901, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 0.6779, 0.3606, -0.1009, -0.0841, -0.0901, -0.1781]), prediction=20.111591250048935), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.16297217562510521, ALT=-0.13685658134190837, STP=-0.13700335699941393, PCP01=-0.17811912685987313, count=15, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.163, -0.1369, -0.137, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.163, -0.1369, -0.137, -0.1781]), prediction=20.111591250048935), Row(SPD=1.3348834165148888, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.089424540891683, DEWP=0.2150046813319293, SLP=-0.1784983922785605, ALT=-0.18959361927604487, STP=-0.18394107952703495, PCP01=-0.17811912685987313, count=15, va_features=DenseVector([1.3349, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.0894, 0.215, -0.1785, -0.1896, -0.1839, -0.1781]), features=DenseVector([1.3349, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.0894, 0.215, -0.1785, -0.1896, -0.1839, -0.1781]), prediction=23.780114059765907), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.2150046813319293, SLP=-0.25612947554581933, ALT=-0.24233065721016261, STP=-0.24652470956386888, PCP01=-0.17811912685987313, count=17, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.215, -0.2561, -0.2423, -0.2465, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.215, -0.2561, -0.2423, -0.2465, -0.1781]), prediction=21.326904911897515), Row(SPD=0.4111950573077732, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.2150046813319293, SLP=-0.3337605588130782, ALT=-0.34780473307841686, STP=-0.34040015461912865, PCP01=-0.17811912685987313, count=12, va_features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.215, -0.3338, -0.3478, -0.3404, -0.1781]), features=DenseVector([0.4112, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.215, -0.3338, -0.3478, -0.3404, -0.1781]), prediction=22.693524979302715), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=0, SCT=1, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.14219939028218284, SLP=-0.3182343421596229, ALT=-0.2950676951442991, STP=-0.29346243209150763, PCP01=-0.17811912685987313, count=27, va_features=DenseVector([-0.2046, -0.3045, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.1422, -0.3182, -0.2951, -0.2935, -0.1781]), features=DenseVector([-0.2046, -0.3045, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.1422, -0.3182, -0.2951, -0.2935, -0.1781]), prediction=26.835392583369064), Row(SPD=-0.2045971821636372, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.3363352496737684, DEWP=0.06939409923243636, SLP=-0.39586542542688175, ALT=-0.4005417710125534, STP=-0.4029837846559626, PCP01=-0.17811912685987313, count=47, va_features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.0694, -0.3959, -0.4005, -0.403, -0.1781]), features=DenseVector([-0.2046, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.3363, 0.0694, -0.3959, -0.4005, -0.403, -0.1781]), prediction=21.326904911897515), Row(SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.25403168007974, DEWP=0.06939409923243636, SLP=-0.44244407538722996, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=165, va_features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.0694, -0.4424, -0.4533, -0.4499, -0.1781]), features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.254, 0.0694, -0.4424, -0.4533, -0.4499, -0.1781]), prediction=22.484917237808524), Row(SPD=0.7190911770434785, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.28780997238167577, SLP=-0.47349650869414056, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=128, va_features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.4735, -0.4533, -0.4499, -0.1781]), features=DenseVector([0.7191, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.2878, -0.4735, -0.4533, -0.4499, -0.1781]), prediction=22.484917237808524), Row(SPD=1.0269872967791835, GUS=-0.3045476160907193, CLR=1, SCT=0, BKN=0, OVC=0, OBS=0, POB=0, VSB=0.4638614904960641, TEMP=1.1717281104857113, DEWP=0.06939409923243636, SLP=-0.4579702920406853, ALT=-0.45327880894668987, STP=-0.44992150718360135, PCP01=-0.17811912685987313, count=36, va_features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.0694, -0.458, -0.4533, -0.4499, -0.1781]), features=DenseVector([1.027, -0.3045, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.4639, 1.1717, 0.0694, -0.458, -0.4533, -0.4499, -0.1781]), prediction=22.484917237808524)]
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+
|                SPD|                GUS|CLR|SCT|BKN|OVC|OBS|POB|               VSB|                TEMP|               DEWP|                 SLP|                 ALT|                 STP|               PCP01|count|         va_features|            features|        prediction|
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.18408527535736915|0.06939409923243636|  -0.411391642080337| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313|    2|[-1.1282855413707...|[-1.1282855413707...| 17.95700665994376|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.10178170576334064|0.14219939028218284| -0.3492867754665158|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313|    1|[-0.2045971821636...|[-0.2045971821636...| 17.28247725096573|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.01947813616931213|0.14219939028218284| -0.2871819088527123| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313|    1|[0.71909117704347...|[0.71909117704347...| 18.70006531503304|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.01947813616931213|0.06939409923243636|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|    0|[-1.1282855413707...|[-1.1282855413707...|18.188117222794812|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-0.06282543342471639|0.06939409923243636|-0.24060325889236406|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|    1|[-0.2045971821636...|[-0.2045971821636...|18.188117222794812|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| -0.1451290030187449|0.14219939028218284|-0.20955082558545346|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|    8|[0.71909117704347...|[0.71909117704347...|19.520817025119957|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|-0.06282543342471639| 0.2150046813319293|-0.19402460893199816|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313|   16|[-0.2045971821636...|[-0.2045971821636...| 17.28247725096573|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641| 0.10178170576334064| 0.2150046813319293|-0.14744595897164992|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313|   52|[-0.2045971821636...|[-0.2045971821636...| 17.28247725096573|
|-1.1282855413707529|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.3486924145454262|0.36061526343142225|-0.13191974231819462|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313|   69|[-1.1282855413707...|[-1.1282855413707...|20.111591250048935|
| 1.9506756559862992|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.5132995537334832|0.28780997238167577|-0.08534109235784636|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313|   54|[1.95067565598629...|[1.95067565598629...|22.180979483401217|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  0.6779066929215403|0.36061526343142225|-0.10086730901130166|-0.08411954340779061|-0.09006563447177515|-0.17811912685987313|   21|[0.41119505730777...|[0.41119505730777...|20.111591250048935|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.1717281104857113|0.28780997238167577|-0.16297217562510521|-0.13685658134190837|-0.13700335699941393|-0.17811912685987313|   15|[0.41119505730777...|[0.41119505730777...|20.111591250048935|
| 1.3348834165148888|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|   1.089424540891683| 0.2150046813319293| -0.1784983922785605|-0.18959361927604487|-0.18394107952703495|-0.17811912685987313|   15|[1.33488341651488...|[1.33488341651488...|23.780114059765907|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.3363352496737684| 0.2150046813319293|-0.25612947554581933|-0.24233065721016261|-0.24652470956386888|-0.17811912685987313|   17|[-0.2045971821636...|[-0.2045971821636...|21.326904911897515|
| 0.4111950573077732|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|    1.25403168007974| 0.2150046813319293| -0.3337605588130782|-0.34780473307841686|-0.34040015461912865|-0.17811912685987313|   12|[0.41119505730777...|[0.41119505730777...|22.693524979302715|
|-0.2045971821636372|-0.3045476160907193|  0|  1|  0|  0|  0|  0|0.4638614904960641|  1.3363352496737684|0.14219939028218284| -0.3182343421596229| -0.2950676951442991|-0.29346243209150763|-0.17811912685987313|   27|[-0.2045971821636...|[-0.2045971821636...|26.835392583369064|
|-0.2045971821636372|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.3363352496737684|0.06939409923243636|-0.39586542542688175| -0.4005417710125534| -0.4029837846559626|-0.17811912685987313|   47|[-0.2045971821636...|[-0.2045971821636...|21.326904911897515|
| 1.0269872967791835|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|    1.25403168007974|0.06939409923243636|-0.44244407538722996|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313|  165|[1.02698729677918...|[1.02698729677918...|22.484917237808524|
| 0.7190911770434785|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.1717281104857113|0.28780997238167577|-0.47349650869414056|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313|  128|[0.71909117704347...|[0.71909117704347...|22.484917237808524|
| 1.0269872967791835|-0.3045476160907193|  1|  0|  0|  0|  0|  0|0.4638614904960641|  1.1717281104857113|0.06939409923243636| -0.4579702920406853|-0.45327880894668987|-0.44992150718360135|-0.17811912685987313|   36|[1.02698729677918...|[1.02698729677918...|22.484917237808524|
+-------------------+-------------------+---+---+---+---+---+---+------------------+--------------------+-------------------+--------------------+--------------------+--------------------+--------------------+-----+--------------------+--------------------+------------------+
only showing top 20 rows

df_rf = july_prediction3_df.select("*").toPandas()
df_rf.to_csv('df_rf.csv')
df_rf

	SPD	GUS	CLR	SCT	BKN	OVC	OBS	POB	VSB	TEMP	DEWP	SLP	ALT	STP	PCP01	count	va_features	features	prediction
0	-1.128286	-0.304548	1	0	0	0	0	0	0.463861	0.184085	0.069394	-0.411392	-0.400542	-0.402984	-0.178119	2	[-1.1282855413707529, -0.3045476160907193, 1.0...	[-1.1282855413707529, -0.3045476160907193, 1.0...	17.957007
1	-0.204597	-0.304548	1	0	0	0	0	0	0.463861	0.101782	0.142199	-0.349287	-0.347805	-0.340400	-0.178119	1	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	17.282477
2	0.719091	-0.304548	1	0	0	0	0	0	0.463861	0.019478	0.142199	-0.287182	-0.295068	-0.293462	-0.178119	1	[0.7190911770434785, -0.3045476160907193, 1.0,...	[0.7190911770434785, -0.3045476160907193, 1.0,...	18.700065
3	-1.128286	-0.304548	1	0	0	0	0	0	0.463861	0.019478	0.069394	-0.256129	-0.242331	-0.246525	-0.178119	0	[-1.1282855413707529, -0.3045476160907193, 1.0...	[-1.1282855413707529, -0.3045476160907193, 1.0...	18.188117
4	-0.204597	-0.304548	1	0	0	0	0	0	0.463861	-0.062825	0.069394	-0.240603	-0.242331	-0.246525	-0.178119	1	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	18.188117
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
727	-0.204597	-0.304548	1	0	0	0	0	0	0.097945	0.924817	1.161473	0.302814	0.337777	0.332374	-0.178119	61	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	19.637665
728	-1.128286	-0.304548	1	0	0	0	0	0	-0.349287	1.089425	1.161473	0.333867	0.337777	0.332374	-0.178119	103	[-1.1282855413707529, -0.3045476160907193, 1.0...	[-1.1282855413707529, -0.3045476160907193, 1.0...	20.346083
729	-1.128286	-0.304548	1	0	0	0	0	0	-0.349287	1.171728	1.161473	0.333867	0.337777	0.332374	-0.178119	64	[-1.1282855413707529, -0.3045476160907193, 1.0...	[-1.1282855413707529, -0.3045476160907193, 1.0...	20.346083
730	-0.204597	-0.304548	1	0	0	0	0	0	-0.349287	1.418639	1.161473	0.380445	0.390514	0.394957	-0.178119	20	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	21.125246
731	-0.204597	-0.304548	1	0	0	0	0	0	0.463861	1.418639	1.088668	0.364919	0.390514	0.394957	-0.178119	12	[-0.2045971821636372, -0.3045476160907193, 1.0...	[-0.2045971821636372, -0.3045476160907193, 1.0...	22.378931

732 rows × 19 columns

%matplotlib inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import geopandas as gpd
import folium
import datetime
from folium.plugins import HeatMapWithTime
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation
from IPython.display import HTML

import warnings
warnings.filterwarnings('ignore')

# load data and preprocess datetime
citibike = pd.read_csv('201907-citibike-tripdata.csv')

citibike['starttime'] = citibike['starttime'].str[:-5]
citibike['stoptime'] = citibike['stoptime'].str[:-5]
citibike['starttime'] = pd.to_datetime(citibike['starttime'])
citibike['stoptime'] = pd.to_datetime(citibike['stoptime'])

citibike = citibike.set_index('starttime')

# create list to store lists of locations day by day, hour by hour
df_hour_list = []
day_list = list(set(citibike.index.date))
day_list.sort()
time_index = []
for day in day_list:
    for hour in range(0,24):
        time_index.append(datetime.datetime.combine(day, datetime.time(hour)))
for day in day_list:
    for hour in range(0,24):
        citibike_day = citibike.loc[citibike.index.date == day, ['start station latitude', 'start station longitude']]
        citibike_hour = citibike_day.loc[citibike_day.index.hour == hour, ['start station latitude', 'start station longitude']].groupby(['start station latitude', 'start station longitude']).sum().reset_index().values.tolist()
        citibike_demand = citibike_day.loc[citibike_day.index.hour == hour, ['start station latitude', 'start station longitude']].groupby(['start station latitude', 'start station longitude']).size()
        demand_max = citibike_demand.values.max()
        demand_scaled = citibike_demand.values/demand_max
        for k in range(len(citibike_hour)):
            citibike_hour[k].append(demand_scaled[k])
        df_hour_list.append(citibike_hour)

# add trip events to the map
time_index = [str(x) for x in time_index]
map_time = folium.Map(location=[40.7470, -73.9955], tiles='CartoDB Positron', zoom_start=13)

HeatMapWithTime(df_hour_list, index=time_index, auto_play=True, max_opacity=0.5, gradient={0.2: 'cornflowerblue', 0.4: 'royalblue', 0.75: 'mediumblue', 1: 'blue'}).add_to(map_time)

map_time